From b6f88b9517e244b1659563a9410bcb80cbe59eb2 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 19 Nov 2025 14:56:32 +0100 Subject: [PATCH] Modify ReplaceRegexTool to become ReplaceContentTool, which is more fail-safe and more flexible * It is more fail-safe because it requires no escaping in replacement strings (I have opted to use a different character sequence for backreferences, such that escaping backslashes is no longer required. * It is more flexible, because it adds the "literal" mode, which is simpler to use for models that are not good at generating regexes correctly --- docs/01-about/035_tools.md | 2 +- .../resources/config/contexts/chatgpt.yml | 6 +- .../resources/config/contexts/codex.yml | 2 +- .../config/contexts/ide-assistant.yml | 2 +- src/serena/resources/config/modes/editing.yml | 4 +- .../resources/config/modes/planning.yml | 2 +- src/serena/tools/file_tools.py | 70 ++++++++++++------- src/serena/tools/memory_tools.py | 6 +- 8 files changed, 57 insertions(+), 37 deletions(-) diff --git a/docs/01-about/035_tools.md b/docs/01-about/035_tools.md index 17b84b53..55ff8bf2 100644 --- a/docs/01-about/035_tools.md +++ b/docs/01-about/035_tools.md @@ -35,7 +35,7 @@ Note that in most configurations, only a subset of these tools will be enabled s * `remove_project`: Removes a project from the Serena configuration. * `rename_symbol`: Renames a symbol throughout the codebase using language server refactoring capabilities. * `replace_lines`: Replaces a range of lines within a file with new content. -* `replace_regex`: Replaces content in a file by using regular expressions. +* `replace_content`: Replaces content in a file (optionally using regular expressions). * `replace_symbol_body`: Replaces the full definition of a symbol. * `restart_language_server`: Restarts the language server, may be necessary when edits not through Serena happen. * `search_for_pattern`: Performs a search for a pattern in the project. diff --git a/src/serena/resources/config/contexts/chatgpt.yml b/src/serena/resources/config/contexts/chatgpt.yml index b3780f4c..4a035516 100644 --- a/src/serena/resources/config/contexts/chatgpt.yml +++ b/src/serena/resources/config/contexts/chatgpt.yml @@ -18,9 +18,9 @@ tool_description_overrides: find_symbol: | Retrieves symbols matching `name_path_pattern` in a file. Use `depth > 0` to include children. `name_path_pattern` can be: "foo": any symbol named "foo"; "foo/bar": "bar" within "foo"; "/foo/bar": only top-level "foo/bar" - replace_regex: | - Replaces text using regular expressions. Preferred for smaller edits where symbol-level tools aren't appropriate. - Use wildcards (.*?) to match large sections efficiently: "beginning.*?end" instead of specifying exact content. + replace_content: | + Replaces content in files. Preferred for smaller edits where symbol-level tools aren't appropriate. + Use mode "regex" with wildcards (.*?) to match large sections efficiently: "beginning.*?end" instead of specifying exact content. Essential for multi-line replacements. search_for_pattern: | Flexible pattern search across codebase. Prefer symbolic operations when possible. diff --git a/src/serena/resources/config/contexts/codex.yml b/src/serena/resources/config/contexts/codex.yml index 8ca5433f..2088ab32 100644 --- a/src/serena/resources/config/contexts/codex.yml +++ b/src/serena/resources/config/contexts/codex.yml @@ -21,7 +21,7 @@ excluded_tools: - read_file - execute_shell_command - prepare_for_new_conversation - - replace_regex + - replace_content tool_description_overrides: {} diff --git a/src/serena/resources/config/contexts/ide-assistant.yml b/src/serena/resources/config/contexts/ide-assistant.yml index 78b865b5..8d64541c 100644 --- a/src/serena/resources/config/contexts/ide-assistant.yml +++ b/src/serena/resources/config/contexts/ide-assistant.yml @@ -21,7 +21,7 @@ excluded_tools: - read_file - execute_shell_command - prepare_for_new_conversation - - replace_regex + - replace_content tool_description_overrides: {} diff --git a/src/serena/resources/config/modes/editing.yml b/src/serena/resources/config/modes/editing.yml index b82603b1..ef62c2b7 100644 --- a/src/serena/resources/config/modes/editing.yml +++ b/src/serena/resources/config/modes/editing.yml @@ -34,7 +34,7 @@ prompt: | You will generally be able to use the info from the snippets and the regex-based approach to adjust the references as well. You can assume that all symbol editing tools are reliable, so you don't need to verify the results if the tool returns without error. - {% if 'replace_regex' in available_tools %} + {% if 'replace_content' in available_tools %} Let us discuss the regex-based approach. The regex-based approach is your primary tool for editing code whenever replacing or deleting a whole symbol would be a more expensive operation. This is the case if you need to adjust just a few lines of code within a method, or a chunk that is much smaller than a whole symbol. @@ -73,7 +73,7 @@ prompt: | ``` and you want to replace `x = relu(x)` with `x = gelu(x)`. - You first try `replace_regex()` with the regex `x = relu\(x\)` and the replacement `x = gelu(x)`. + You first try `replace_content` with the regex `x = relu\(x\)` and the replacement `x = gelu(x)`. If this fails due to multiple matches, you will try `(linear\(x\)\s*)x = relu\(x\)(\s*return)` with the replacement `\1x = gelu(x)\2`. 2 Larger replacement diff --git a/src/serena/resources/config/modes/planning.yml b/src/serena/resources/config/modes/planning.yml index a3f76d79..ce0bf2e1 100644 --- a/src/serena/resources/config/modes/planning.yml +++ b/src/serena/resources/config/modes/planning.yml @@ -12,4 +12,4 @@ excluded_tools: - replace_lines - insert_at_line - execute_shell_command - - replace_regex + - replace_content diff --git a/src/serena/tools/file_tools.py b/src/serena/tools/file_tools.py index 1d091535..96b58907 100644 --- a/src/serena/tools/file_tools.py +++ b/src/serena/tools/file_tools.py @@ -12,6 +12,7 @@ import re from collections import defaultdict from fnmatch import fnmatch from pathlib import Path +from typing import Literal from serena.text_utils import search_files from serena.tools import SUCCESS_RESULT, EditedFileContext, Tool, ToolMarkerCanEdit, ToolMarkerOptional @@ -157,72 +158,91 @@ class FindFileTool(Tool): return result -class ReplaceRegexTool(Tool, ToolMarkerCanEdit): +class ReplaceContentTool(Tool, ToolMarkerCanEdit): """ - Replaces content in a file by using regular expressions. + Replaces content in a file (optionally using regular expressions). """ def apply( self, relative_path: str, - regex: str, + needle: str, repl: str, + mode: Literal["literal", "regex"], allow_multiple_occurrences: bool = False, ) -> str: r""" - Replaces one or more occurrences of the given regular expression. - Works exactly like Python's re.subn function with flags DOTALL and MULTILINE enabled. + Replaces one or more occurrences of a given pattern in a file with new content. This is the preferred way to replace content in a file whenever the symbol-level tools are not appropriate. - Even large sections of code can be replaced by providing a concise regular expression of - the form "beginning.*?end-of-text-to-be-replaced". - Always try to use wildcards to avoid specifying the exact content of the code to be replaced, - especially if it spans several lines. - IMPORTANT: REMEMBER TO USE WILDCARDS WHEN APPROPRIATE! I WILL BE VERY UNHAPPY IF YOU WRITE UNNECESSARILY LONG REGEXES WITHOUT USING WILDCARDS! + VERY IMPORTANT: The "regex" mode allows very large sections of code to be replaced without fully quoting them! + Use a regex of the form "beginning.*?end-of-text-to-be-replaced" to be faster and more economical! + ALWAYS try to use wildcards to avoid specifying the exact content to be replaced, + especially if it spans several lines. Note that you cannot make mistakes, because if the regex should match + multiple occurrences while you disabled `allow_multiple_occurrences`, an error will be returned, and you can retry + with a revised regex. :param relative_path: the relative path to the file - :param regex: a Python-style regular expression, matches of which will be replaced. - Dot matches all characters, multi-line matching is enabled. - Apply the usual escaping as needed for reserved characters in Python-style regex. - :param repl: the string to replace the matched content with, which may contain - backreferences like \1, \2, etc. for groups matched by the regex. - Insert new content verbatim, except for backslashes, which have to be escaped. - IMPORTANT: When inserting a quoted string with newlines encoded as backslash followed by n, don't forget to insert double backslashes (i.e., \\n). + :param needle: the string or regex pattern to search for. + If `mode` is "literal", this string will be matched exactly. + If `mode` is "regex", this string will be treated as a regular expression (syntax of Python's `re` module, + with flags DOTALL and MULTILINE enabled). + :param repl: the replacement string (verbatim). + If mode is "regex", the string can contain backreferences to matched groups in the needle regex, + specified using the syntax $!1, $!2, etc. for groups 1, 2, etc. + :param mode: either "literal" or "regex", specifying how the `needle` parameter is to be interpreted. :param allow_multiple_occurrences: if True, the regex may match multiple occurrences in the file and all of them will be replaced. If this is set to False and the regex matches multiple occurrences, an error will be returned (and you may retry with a revised, more specific regex). """ - return self.replace_regex( - relative_path, regex, repl, allow_multiple_occurrences=allow_multiple_occurrences, require_not_ignored=True + return self.replace_content( + relative_path, needle, repl, mode=mode, allow_multiple_occurrences=allow_multiple_occurrences, require_not_ignored=True ) - def replace_regex( + def replace_content( self, relative_path: str, - regex: str, + needle: str, repl: str, + mode: Literal["literal", "regex"], allow_multiple_occurrences: bool = False, require_not_ignored: bool = True, ) -> str: """ - Performs the regex replacement, with additional options not exposed in the tool. + Performs the replacement, with additional options not exposed in the tool. This function can be used internally by other tools. """ self.project.validate_relative_path(relative_path, require_not_ignored=require_not_ignored) with EditedFileContext(relative_path, self.agent) as context: original_content = context.get_original_content() + + if mode == "literal": + regex = re.escape(needle) + elif mode == "regex": + regex = needle + else: + raise ValueError(f"Invalid mode: '{mode}', expected 'literal' or 'regex'.") + + # escape backslashes in repl + repl = repl.replace("\\", "\\\\") + # convert $!1, $!2, etc. to \1, \2, etc. for backreferences + repl = re.sub(r"\$!(\d+)", "\\\\1", repl) + + # perform replacement updated_content, n = re.subn(regex, repl, original_content, flags=re.DOTALL | re.MULTILINE) + if n == 0: - return f"Error: No matches found for regex '{regex}' in file '{relative_path}'." + return f"Error: No matches of search expression found in file '{relative_path}'." if not allow_multiple_occurrences and n > 1: return ( - f"Error: Regex '{regex}' matches {n} occurrences in file '{relative_path}'. " - "Please revise the regex to be more specific or enable allow_multiple_occurrences if this is expected." + f"Error: Expression matches {n} occurrences in file '{relative_path}'. " + "Please revise the expression to be more specific or enable allow_multiple_occurrences if this is expected." ) context.set_updated_content(updated_content) + return SUCCESS_RESULT diff --git a/src/serena/tools/memory_tools.py b/src/serena/tools/memory_tools.py index c6e2b93b..e9280dc1 100644 --- a/src/serena/tools/memory_tools.py +++ b/src/serena/tools/memory_tools.py @@ -1,6 +1,6 @@ import json -from serena.tools import ReplaceRegexTool, Tool +from serena.tools import ReplaceContentTool, Tool class WriteMemoryTool(Tool): @@ -84,6 +84,6 @@ class EditMemoryTool(Tool): backreferences like \1, \2, etc. for groups matched by the regex. Insert new content verbatim, except for backslashes, which have to be escaped. """ - replace_regex_tool = self.agent.get_tool(ReplaceRegexTool) + replace_content_tool = self.agent.get_tool(ReplaceContentTool) rel_path = self.memories_manager.get_memory_file_path(memory_file_name).relative_to(self.get_project_root()) - return replace_regex_tool.replace_regex(str(rel_path), regex, repl, require_not_ignored=False) + return replace_content_tool.replace_content(str(rel_path), regex, repl, mode="regex", require_not_ignored=False)