Merge pull request #774 from oraios/new-replace-tool

More fail-safe and flexible content replacement tool

CHANGELOG.md

@@ -17,6 +17,8 @@ Status of the `main` branch. Changes prior to the next official version change w
 
 * Tools:
   * Added `RenameSymbolTool` for renaming symbols across the codebase (if LS supports this operation).
+  * Replaced `ReplaceRegexTool` with `ReplaceContentTool`, which supports both plain text and regex-based replacements
+    (and which requires no escaping in the replacement text, making it more robust) 
 
 * Language support:
 

docs/01-about/035_tools.md

@@ -35,7 +35,7 @@ Note that in most configurations, only a subset of these tools will be enabled s
 * `remove_project`: Removes a project from the Serena configuration.
 * `rename_symbol`: Renames a symbol throughout the codebase using language server refactoring capabilities.
 * `replace_lines`: Replaces a range of lines within a file with new content.
-* `replace_regex`: Replaces content in a file by using regular expressions.
+* `replace_content`: Replaces content in a file (optionally using regular expressions).
 * `replace_symbol_body`: Replaces the full definition of a symbol.
 * `restart_language_server`: Restarts the language server, may be necessary when edits not through Serena happen.
 * `search_for_pattern`: Performs a search for a pattern in the project.

src/serena/config/serena_config.py

@@ -33,6 +33,7 @@ from serena.util.inspection import determine_programming_language_composition
 from solidlsp.ls_config import Language
 
 from ..analytics import RegisteredTokenCountEstimator
+from ..tools import ReplaceContentTool
 from ..util.class_decorators import singleton
 
 if TYPE_CHECKING:
@@ -70,6 +71,11 @@ class SerenaPaths:
 
 
 class ToolSet:
+    LEGACY_TOOL_NAME_MAPPING = {"replace_regex": ReplaceContentTool.get_name_from_cls()}
+    """
+    maps legacy tool names to their new names for backward compatibility
+    """
+
     def __init__(self, tool_names: set[str]) -> None:
         self._tool_names = tool_names
 
@@ -89,18 +95,28 @@ class ToolSet:
         """
         from serena.tools import ToolRegistry
 
+        def get_updated_tool_name(tool_name: str) -> str:
+            """Retrieves the updated tool name if the provided tool name is deprecated, logging a warning."""
+            if tool_name in self.LEGACY_TOOL_NAME_MAPPING:
+                new_tool_name = self.LEGACY_TOOL_NAME_MAPPING[tool_name]
+                log.warning("Tool name '%s' is deprecated, please use '%s' instead", tool_name, new_tool_name)
+                return new_tool_name
+            return tool_name
+
         registry = ToolRegistry()
         tool_names = set(self._tool_names)
         for definition in tool_inclusion_definitions:
             included_tools = []
             excluded_tools = []
             for included_tool in definition.included_optional_tools:
+                included_tool = get_updated_tool_name(included_tool)
                 if not registry.is_valid_tool_name(included_tool):
                     raise ValueError(f"Invalid tool name '{included_tool}' provided for inclusion")
                 if included_tool not in tool_names:
                     tool_names.add(included_tool)
                     included_tools.append(included_tool)
             for excluded_tool in definition.excluded_tools:
+                excluded_tool = get_updated_tool_name(excluded_tool)
                 if not registry.is_valid_tool_name(excluded_tool):
                     raise ValueError(f"Invalid tool name '{excluded_tool}' provided for exclusion")
                 if excluded_tool in tool_names:

src/serena/resources/config/contexts/chatgpt.yml

@@ -18,9 +18,9 @@ tool_description_overrides:
   find_symbol: |
     Retrieves symbols matching `name_path_pattern` in a file.
     Use `depth > 0` to include children. `name_path_pattern` can be: "foo": any symbol named "foo"; "foo/bar": "bar" within "foo"; "/foo/bar": only top-level "foo/bar"
-  replace_regex: |
-    Replaces text using regular expressions. Preferred for smaller edits where symbol-level tools aren't appropriate.
-    Use wildcards (.*?) to match large sections efficiently: "beginning.*?end" instead of specifying exact content.
+  replace_content: |
+    Replaces content in files. Preferred for smaller edits where symbol-level tools aren't appropriate.
+    Use mode "regex" with wildcards (.*?) to match large sections efficiently: "beginning.*?end" instead of specifying exact content.
     Essential for multi-line replacements.
   search_for_pattern: |
     Flexible pattern search across codebase. Prefer symbolic operations when possible.

src/serena/resources/config/contexts/codex.yml

@@ -21,7 +21,7 @@ excluded_tools:
   - read_file
   - execute_shell_command
   - prepare_for_new_conversation
-  - replace_regex
+  - replace_content
 
 
 tool_description_overrides: {}

src/serena/resources/config/contexts/ide-assistant.yml

@@ -21,7 +21,7 @@ excluded_tools:
   - read_file
   - execute_shell_command
   - prepare_for_new_conversation
-  - replace_regex
+  - replace_content
 
 tool_description_overrides: {}
 

src/serena/resources/config/modes/editing.yml

@@ -8,14 +8,12 @@ prompt: |
   When writing new code, think about where it belongs best. Don't generate new files if you don't plan on actually
   integrating them into the codebase, instead use the editing tools to insert the code directly into the existing files in that case.
 
-  You have two main approaches for editing code - editing by regex and editing by symbol.
+  You have two main approaches for editing code - editing at the symbol level and more file-based editing.
   The symbol-based approach is appropriate if you need to adjust an entire symbol, e.g. a method, a class, a function, etc.
-  But it is not appropriate if you need to adjust just a few lines of code within a symbol, for that you should
-  use the regex-based approach that is described below.
+  But it is not appropriate if you need to adjust just a few lines of code within a symbol.
 
   Let us first discuss the symbol-based approach.
-  Symbols are identified by their name path and relative file path, see the description of the `find_symbol` tool for more details
-  on how the `name_path` matches symbols.
+  Symbols are identified by their name path and relative file path (description of the `find_symbol` tool).
   You can get information about available symbols by using the `get_symbols_overview` tool for finding top-level symbols in a file,
   or by using `find_symbol` if you already know the symbol's name path. You generally try to read as little code as possible
   while still solving your task, meaning you only read the bodies when you need to, and after you have found the symbol you want to edit.
@@ -34,77 +32,12 @@ prompt: |
   You will generally be able to use the info from the snippets and the regex-based approach to adjust the references as well.
   You can assume that all symbol editing tools are reliable, so you don't need to verify the results if the tool returns without error.
 
-  {% if 'replace_regex' in available_tools %}
-  Let us discuss the regex-based approach.
-  The regex-based approach is your primary tool for editing code whenever replacing or deleting a whole symbol would be a more expensive operation.
-  This is the case if you need to adjust just a few lines of code within a method, or a chunk that is much smaller than a whole symbol.
-  You use other tools to find the relevant content and
-  then use your knowledge of the codebase to write the regex, if you haven't collected enough information of this content yet.
+  {% if 'replace_content' in available_tools %}
+  The `replace_content` tool allows you to perform regex-based replacements within files.
+  This is your primary tool for editing code whenever replacing or deleting a whole symbol would be a more expensive operation,
+  e.g. if you need to adjust just a few lines of code within a method.
   You are extremely good at regex, so you never need to check whether the replacement produced the correct result.
-  In particular, you know what to escape and what not to escape, and you know how to use wildcards.
-  Also, the regex tool never adds any indentation (contrary to the symbolic editing tools), so you have to take care to add the correct indentation
-  when using it to insert code.
-  Moreover, the replacement tool will fail if it can't perform the desired replacement, and this is all the feedback you need.
-  Your overall goal for replacement operations is to use relatively short regexes, since I want you to minimize the number
-  of output tokens. For replacements of larger chunks of code, this means you intelligently make use of wildcards for the middle part 
-  and of characteristic snippets for the before/after parts that uniquely identify the chunk.
-  
-  For small replacements, up to a single line, you follow the following rules:
-
-    1. If the snippet to be replaced is likely to be unique within the file, you perform the replacement by directly using the escaped version of the 
-       original.
-    2. If the snippet is probably not unique, and you want to replace all occurrences, you use the `allow_multiple_occurrences` flag.
-    3. If the snippet is not unique, and you want to replace a specific occurrence, you make use of the code surrounding the snippet
-       to extend the regex with content before/after such that the regex will have exactly one match.
-    4. You generally assume that a snippet is unique, knowing that the tool will return an error on multiple matches. You only read more file content
-       (for crafvarting a more specific regex) if such a failure unexpectedly occurs. 
-
-  Examples:
-
-  1 Small replacement
-  You have read code like
-    
-    ```python
-    ...
-    x = linear(x)
-    x = relu(x)
-    return x
-    ...
-    ```
-
-  and you want to replace `x = relu(x)` with `x = gelu(x)`.
-  You first try `replace_regex()` with the regex `x = relu\(x\)` and the replacement `x = gelu(x)`.
-  If this fails due to multiple matches, you will try `(linear\(x\)\s*)x = relu\(x\)(\s*return)` with the replacement `\1x = gelu(x)\2`.
-
-  2 Larger replacement
-
-  You have read code like
-
-  ```python
-  def my_func():
-    ...
-    # a comment before the snippet
-    x = add_fifteen(x)
-    # beginning of long section within my_func
-    ....
-    # end of long section
-    call_subroutine(z)
-    call_second_subroutine(z)
-  ```
-  and you want to replace the code starting with `x = add_fifteen(x)` until (including) `call_subroutine(z)`, but not `call_second_subroutine(z)`.
-  Initially, you assume that the the beginning and end of the chunk uniquely determine it within the file.
-  Therefore, you perform the replacement by using the regex `x = add_fifteen\(x\)\s*.*?call_subroutine\(z\)`
-  and the replacement being the new code you want to insert.
-
-  If this fails due to multiple matches, you will try to extend the regex with the content before/after the snippet and match groups. 
-  The matching regex becomes:
-  `(before the snippet\s*)x = add_fifteen\(x\)\s*.*?call_subroutine\(z\)` 
-  and the replacement includes the group as (schematically):
-  `\1<new_code>`
-
-  Generally, I remind you that you rely on the regex tool with providing you the correct feedback, no need for more verification!
-
-  IMPORTANT: REMEMBER TO USE WILDCARDS WHEN APPROPRIATE! I WILL BE VERY UNHAPPY IF YOU WRITE LONG REGEXES WITHOUT USING WILDCARDS INSTEAD!
+  In particular, you know how to use wildcards effectively in order to avoid specifying the full original text to be replaced!
   {% endif %}
 excluded_tools:
  - replace_lines

src/serena/resources/config/modes/planning.yml

@@ -12,4 +12,4 @@ excluded_tools:
   - replace_lines
   - insert_at_line
   - execute_shell_command
-  - replace_regex
+  - replace_content

src/serena/tools/file_tools.py

@@ -12,6 +12,7 @@ import re
 from collections import defaultdict
 from fnmatch import fnmatch
 from pathlib import Path
+from typing import Literal
 
 from serena.text_utils import search_files
 from serena.tools import SUCCESS_RESULT, EditedFileContext, Tool, ToolMarkerCanEdit, ToolMarkerOptional
@@ -157,72 +158,92 @@ class FindFileTool(Tool):
         return result
 
 
-class ReplaceRegexTool(Tool, ToolMarkerCanEdit):
+class ReplaceContentTool(Tool, ToolMarkerCanEdit):
     """
-    Replaces content in a file by using regular expressions.
+    Replaces content in a file (optionally using regular expressions).
     """
 
     def apply(
         self,
         relative_path: str,
-        regex: str,
+        needle: str,
         repl: str,
+        mode: Literal["literal", "regex"],
         allow_multiple_occurrences: bool = False,
     ) -> str:
         r"""
-        Replaces one or more occurrences of the given regular expression.
-        Works exactly like Python's re.subn function with flags DOTALL and MULTILINE enabled.
+        Replaces one or more occurrences of a given pattern in a file with new content.
 
         This is the preferred way to replace content in a file whenever the symbol-level
         tools are not appropriate.
-        Even large sections of code can be replaced by providing a concise regular expression of
-        the form "beginning.*?end-of-text-to-be-replaced".
-        Always try to use wildcards to avoid specifying the exact content of the code to be replaced,
-        especially if it spans several lines.
 
-        IMPORTANT: REMEMBER TO USE WILDCARDS WHEN APPROPRIATE! I WILL BE VERY UNHAPPY IF YOU WRITE UNNECESSARILY LONG REGEXES WITHOUT USING WILDCARDS!
+        VERY IMPORTANT: The "regex" mode allows very large sections of code to be replaced without fully quoting them!
+        Use a regex of the form "beginning.*?end-of-text-to-be-replaced" to be faster and more economical!
+        ALWAYS try to use wildcards to avoid specifying the exact content to be replaced,
+        especially if it spans several lines. Note that you cannot make mistakes, because if the regex should match
+        multiple occurrences while you disabled `allow_multiple_occurrences`, an error will be returned, and you can retry
+        with a revised regex.
+        Therefore, using regex mode with suitable wildcards is usually the best choice!
 
         :param relative_path: the relative path to the file
-        :param regex: a Python-style regular expression, matches of which will be replaced.
-            Dot matches all characters, multi-line matching is enabled.
-            Apply the usual escaping as needed for reserved characters in Python-style regex.
-        :param repl: the string to replace the matched content with, which may contain
-            backreferences like \1, \2, etc. for groups matched by the regex.
-            Insert new content verbatim, except for backslashes, which have to be escaped.
-            IMPORTANT: When inserting a quoted string with newlines encoded as backslash followed by n, don't forget to insert double backslashes (i.e., \\n).
+        :param needle: the string or regex pattern to search for.
+            If `mode` is "literal", this string will be matched exactly.
+            If `mode` is "regex", this string will be treated as a regular expression (syntax of Python's `re` module,
+            with flags DOTALL and MULTILINE enabled).
+        :param repl: the replacement string (verbatim).
+            If mode is "regex", the string can contain backreferences to matched groups in the needle regex,
+            specified using the syntax $!1, $!2, etc. for groups 1, 2, etc.
+        :param mode: either "literal" or "regex", specifying how the `needle` parameter is to be interpreted.
         :param allow_multiple_occurrences: if True, the regex may match multiple occurrences in the file
             and all of them will be replaced.
             If this is set to False and the regex matches multiple occurrences, an error will be returned
             (and you may retry with a revised, more specific regex).
         """
-        return self.replace_regex(
-            relative_path, regex, repl, allow_multiple_occurrences=allow_multiple_occurrences, require_not_ignored=True
+        return self.replace_content(
+            relative_path, needle, repl, mode=mode, allow_multiple_occurrences=allow_multiple_occurrences, require_not_ignored=True
         )
 
-    def replace_regex(
+    def replace_content(
         self,
         relative_path: str,
-        regex: str,
+        needle: str,
         repl: str,
+        mode: Literal["literal", "regex"],
         allow_multiple_occurrences: bool = False,
         require_not_ignored: bool = True,
     ) -> str:
         """
-        Performs the regex replacement, with additional options not exposed in the tool.
+        Performs the replacement, with additional options not exposed in the tool.
         This function can be used internally by other tools.
         """
         self.project.validate_relative_path(relative_path, require_not_ignored=require_not_ignored)
         with EditedFileContext(relative_path, self.agent) as context:
             original_content = context.get_original_content()
+
+            if mode == "literal":
+                regex = re.escape(needle)
+            elif mode == "regex":
+                regex = needle
+            else:
+                raise ValueError(f"Invalid mode: '{mode}', expected 'literal' or 'regex'.")
+
+            # escape backslashes in repl
+            repl = repl.replace("\\", "\\\\")
+            # convert $!1, $!2, etc. to \1, \2, etc. for backreferences
+            repl = re.sub(r"\$!(\d+)", "\\\\1", repl)
+
+            # perform replacement
             updated_content, n = re.subn(regex, repl, original_content, flags=re.DOTALL | re.MULTILINE)
+
             if n == 0:
-                return f"Error: No matches found for regex '{regex}' in file '{relative_path}'."
+                return f"Error: No matches of search expression found in file '{relative_path}'."
             if not allow_multiple_occurrences and n > 1:
                 return (
-                    f"Error: Regex '{regex}' matches {n} occurrences in file '{relative_path}'. "
-                    "Please revise the regex to be more specific or enable allow_multiple_occurrences if this is expected."
+                    f"Error: Expression matches {n} occurrences in file '{relative_path}'. "
+                    "Please revise the expression to be more specific or enable allow_multiple_occurrences if this is expected."
                 )
             context.set_updated_content(updated_content)
+
         return SUCCESS_RESULT
 
 

src/serena/tools/memory_tools.py

@@ -1,6 +1,7 @@
 import json
+from typing import Literal
 
-from serena.tools import ReplaceRegexTool, Tool
+from serena.tools import ReplaceContentTool, Tool
 
 
 class WriteMemoryTool(Tool):
@@ -70,20 +71,21 @@ class EditMemoryTool(Tool):
     def apply(
         self,
         memory_file_name: str,
-        regex: str,
+        needle: str,
         repl: str,
+        mode: Literal["literal", "regex"],
     ) -> str:
         r"""
         Replaces content matching a regular expression in a memory.
 
         :param memory_file_name: the name of the memory
-        :param regex: a Python-style regular expression (flags DOTALL and MULTILINE enabled, i.e.
-            '.' matches all characters, multi-line matching is enabled).
-            Apply the usual escaping as needed for reserved characters in Python-style regex.
-        :param repl: the string to replace the matched content with, which may contain
-            backreferences like \1, \2, etc. for groups matched by the regex.
-            Insert new content verbatim, except for backslashes, which have to be escaped.
+        :param needle: the string or regex pattern to search for.
+            If `mode` is "literal", this string will be matched exactly.
+            If `mode` is "regex", this string will be treated as a regular expression (syntax of Python's `re` module,
+            with flags DOTALL and MULTILINE enabled).
+        :param repl: the replacement string (verbatim).
+        :param mode: either "literal" or "regex", specifying how the `needle` parameter is to be interpreted.
         """
-        replace_regex_tool = self.agent.get_tool(ReplaceRegexTool)
+        replace_content_tool = self.agent.get_tool(ReplaceContentTool)
         rel_path = self.memories_manager.get_memory_file_path(memory_file_name).relative_to(self.get_project_root())
-        return replace_regex_tool.replace_regex(str(rel_path), regex, repl, require_not_ignored=False)
+        return replace_content_tool.replace_content(str(rel_path), needle, repl, mode=mode, require_not_ignored=False)