From b6f88b9517e244b1659563a9410bcb80cbe59eb2 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 19 Nov 2025 14:56:32 +0100 Subject: [PATCH 1/6] Modify ReplaceRegexTool to become ReplaceContentTool, which is more fail-safe and more flexible * It is more fail-safe because it requires no escaping in replacement strings (I have opted to use a different character sequence for backreferences, such that escaping backslashes is no longer required. * It is more flexible, because it adds the "literal" mode, which is simpler to use for models that are not good at generating regexes correctly --- docs/01-about/035_tools.md | 2 +- .../resources/config/contexts/chatgpt.yml | 6 +- .../resources/config/contexts/codex.yml | 2 +- .../config/contexts/ide-assistant.yml | 2 +- src/serena/resources/config/modes/editing.yml | 4 +- .../resources/config/modes/planning.yml | 2 +- src/serena/tools/file_tools.py | 70 ++++++++++++------- src/serena/tools/memory_tools.py | 6 +- 8 files changed, 57 insertions(+), 37 deletions(-) diff --git a/docs/01-about/035_tools.md b/docs/01-about/035_tools.md index 17b84b53..55ff8bf2 100644 --- a/docs/01-about/035_tools.md +++ b/docs/01-about/035_tools.md @@ -35,7 +35,7 @@ Note that in most configurations, only a subset of these tools will be enabled s * `remove_project`: Removes a project from the Serena configuration. * `rename_symbol`: Renames a symbol throughout the codebase using language server refactoring capabilities. * `replace_lines`: Replaces a range of lines within a file with new content. -* `replace_regex`: Replaces content in a file by using regular expressions. +* `replace_content`: Replaces content in a file (optionally using regular expressions). * `replace_symbol_body`: Replaces the full definition of a symbol. * `restart_language_server`: Restarts the language server, may be necessary when edits not through Serena happen. * `search_for_pattern`: Performs a search for a pattern in the project. diff --git a/src/serena/resources/config/contexts/chatgpt.yml b/src/serena/resources/config/contexts/chatgpt.yml index b3780f4c..4a035516 100644 --- a/src/serena/resources/config/contexts/chatgpt.yml +++ b/src/serena/resources/config/contexts/chatgpt.yml @@ -18,9 +18,9 @@ tool_description_overrides: find_symbol: | Retrieves symbols matching `name_path_pattern` in a file. Use `depth > 0` to include children. `name_path_pattern` can be: "foo": any symbol named "foo"; "foo/bar": "bar" within "foo"; "/foo/bar": only top-level "foo/bar" - replace_regex: | - Replaces text using regular expressions. Preferred for smaller edits where symbol-level tools aren't appropriate. - Use wildcards (.*?) to match large sections efficiently: "beginning.*?end" instead of specifying exact content. + replace_content: | + Replaces content in files. Preferred for smaller edits where symbol-level tools aren't appropriate. + Use mode "regex" with wildcards (.*?) to match large sections efficiently: "beginning.*?end" instead of specifying exact content. Essential for multi-line replacements. search_for_pattern: | Flexible pattern search across codebase. Prefer symbolic operations when possible. diff --git a/src/serena/resources/config/contexts/codex.yml b/src/serena/resources/config/contexts/codex.yml index 8ca5433f..2088ab32 100644 --- a/src/serena/resources/config/contexts/codex.yml +++ b/src/serena/resources/config/contexts/codex.yml @@ -21,7 +21,7 @@ excluded_tools: - read_file - execute_shell_command - prepare_for_new_conversation - - replace_regex + - replace_content tool_description_overrides: {} diff --git a/src/serena/resources/config/contexts/ide-assistant.yml b/src/serena/resources/config/contexts/ide-assistant.yml index 78b865b5..8d64541c 100644 --- a/src/serena/resources/config/contexts/ide-assistant.yml +++ b/src/serena/resources/config/contexts/ide-assistant.yml @@ -21,7 +21,7 @@ excluded_tools: - read_file - execute_shell_command - prepare_for_new_conversation - - replace_regex + - replace_content tool_description_overrides: {} diff --git a/src/serena/resources/config/modes/editing.yml b/src/serena/resources/config/modes/editing.yml index b82603b1..ef62c2b7 100644 --- a/src/serena/resources/config/modes/editing.yml +++ b/src/serena/resources/config/modes/editing.yml @@ -34,7 +34,7 @@ prompt: | You will generally be able to use the info from the snippets and the regex-based approach to adjust the references as well. You can assume that all symbol editing tools are reliable, so you don't need to verify the results if the tool returns without error. - {% if 'replace_regex' in available_tools %} + {% if 'replace_content' in available_tools %} Let us discuss the regex-based approach. The regex-based approach is your primary tool for editing code whenever replacing or deleting a whole symbol would be a more expensive operation. This is the case if you need to adjust just a few lines of code within a method, or a chunk that is much smaller than a whole symbol. @@ -73,7 +73,7 @@ prompt: | ``` and you want to replace `x = relu(x)` with `x = gelu(x)`. - You first try `replace_regex()` with the regex `x = relu\(x\)` and the replacement `x = gelu(x)`. + You first try `replace_content` with the regex `x = relu\(x\)` and the replacement `x = gelu(x)`. If this fails due to multiple matches, you will try `(linear\(x\)\s*)x = relu\(x\)(\s*return)` with the replacement `\1x = gelu(x)\2`. 2 Larger replacement diff --git a/src/serena/resources/config/modes/planning.yml b/src/serena/resources/config/modes/planning.yml index a3f76d79..ce0bf2e1 100644 --- a/src/serena/resources/config/modes/planning.yml +++ b/src/serena/resources/config/modes/planning.yml @@ -12,4 +12,4 @@ excluded_tools: - replace_lines - insert_at_line - execute_shell_command - - replace_regex + - replace_content diff --git a/src/serena/tools/file_tools.py b/src/serena/tools/file_tools.py index 1d091535..96b58907 100644 --- a/src/serena/tools/file_tools.py +++ b/src/serena/tools/file_tools.py @@ -12,6 +12,7 @@ import re from collections import defaultdict from fnmatch import fnmatch from pathlib import Path +from typing import Literal from serena.text_utils import search_files from serena.tools import SUCCESS_RESULT, EditedFileContext, Tool, ToolMarkerCanEdit, ToolMarkerOptional @@ -157,72 +158,91 @@ class FindFileTool(Tool): return result -class ReplaceRegexTool(Tool, ToolMarkerCanEdit): +class ReplaceContentTool(Tool, ToolMarkerCanEdit): """ - Replaces content in a file by using regular expressions. + Replaces content in a file (optionally using regular expressions). """ def apply( self, relative_path: str, - regex: str, + needle: str, repl: str, + mode: Literal["literal", "regex"], allow_multiple_occurrences: bool = False, ) -> str: r""" - Replaces one or more occurrences of the given regular expression. - Works exactly like Python's re.subn function with flags DOTALL and MULTILINE enabled. + Replaces one or more occurrences of a given pattern in a file with new content. This is the preferred way to replace content in a file whenever the symbol-level tools are not appropriate. - Even large sections of code can be replaced by providing a concise regular expression of - the form "beginning.*?end-of-text-to-be-replaced". - Always try to use wildcards to avoid specifying the exact content of the code to be replaced, - especially if it spans several lines. - IMPORTANT: REMEMBER TO USE WILDCARDS WHEN APPROPRIATE! I WILL BE VERY UNHAPPY IF YOU WRITE UNNECESSARILY LONG REGEXES WITHOUT USING WILDCARDS! + VERY IMPORTANT: The "regex" mode allows very large sections of code to be replaced without fully quoting them! + Use a regex of the form "beginning.*?end-of-text-to-be-replaced" to be faster and more economical! + ALWAYS try to use wildcards to avoid specifying the exact content to be replaced, + especially if it spans several lines. Note that you cannot make mistakes, because if the regex should match + multiple occurrences while you disabled `allow_multiple_occurrences`, an error will be returned, and you can retry + with a revised regex. :param relative_path: the relative path to the file - :param regex: a Python-style regular expression, matches of which will be replaced. - Dot matches all characters, multi-line matching is enabled. - Apply the usual escaping as needed for reserved characters in Python-style regex. - :param repl: the string to replace the matched content with, which may contain - backreferences like \1, \2, etc. for groups matched by the regex. - Insert new content verbatim, except for backslashes, which have to be escaped. - IMPORTANT: When inserting a quoted string with newlines encoded as backslash followed by n, don't forget to insert double backslashes (i.e., \\n). + :param needle: the string or regex pattern to search for. + If `mode` is "literal", this string will be matched exactly. + If `mode` is "regex", this string will be treated as a regular expression (syntax of Python's `re` module, + with flags DOTALL and MULTILINE enabled). + :param repl: the replacement string (verbatim). + If mode is "regex", the string can contain backreferences to matched groups in the needle regex, + specified using the syntax $!1, $!2, etc. for groups 1, 2, etc. + :param mode: either "literal" or "regex", specifying how the `needle` parameter is to be interpreted. :param allow_multiple_occurrences: if True, the regex may match multiple occurrences in the file and all of them will be replaced. If this is set to False and the regex matches multiple occurrences, an error will be returned (and you may retry with a revised, more specific regex). """ - return self.replace_regex( - relative_path, regex, repl, allow_multiple_occurrences=allow_multiple_occurrences, require_not_ignored=True + return self.replace_content( + relative_path, needle, repl, mode=mode, allow_multiple_occurrences=allow_multiple_occurrences, require_not_ignored=True ) - def replace_regex( + def replace_content( self, relative_path: str, - regex: str, + needle: str, repl: str, + mode: Literal["literal", "regex"], allow_multiple_occurrences: bool = False, require_not_ignored: bool = True, ) -> str: """ - Performs the regex replacement, with additional options not exposed in the tool. + Performs the replacement, with additional options not exposed in the tool. This function can be used internally by other tools. """ self.project.validate_relative_path(relative_path, require_not_ignored=require_not_ignored) with EditedFileContext(relative_path, self.agent) as context: original_content = context.get_original_content() + + if mode == "literal": + regex = re.escape(needle) + elif mode == "regex": + regex = needle + else: + raise ValueError(f"Invalid mode: '{mode}', expected 'literal' or 'regex'.") + + # escape backslashes in repl + repl = repl.replace("\\", "\\\\") + # convert $!1, $!2, etc. to \1, \2, etc. for backreferences + repl = re.sub(r"\$!(\d+)", "\\\\1", repl) + + # perform replacement updated_content, n = re.subn(regex, repl, original_content, flags=re.DOTALL | re.MULTILINE) + if n == 0: - return f"Error: No matches found for regex '{regex}' in file '{relative_path}'." + return f"Error: No matches of search expression found in file '{relative_path}'." if not allow_multiple_occurrences and n > 1: return ( - f"Error: Regex '{regex}' matches {n} occurrences in file '{relative_path}'. " - "Please revise the regex to be more specific or enable allow_multiple_occurrences if this is expected." + f"Error: Expression matches {n} occurrences in file '{relative_path}'. " + "Please revise the expression to be more specific or enable allow_multiple_occurrences if this is expected." ) context.set_updated_content(updated_content) + return SUCCESS_RESULT diff --git a/src/serena/tools/memory_tools.py b/src/serena/tools/memory_tools.py index c6e2b93b..e9280dc1 100644 --- a/src/serena/tools/memory_tools.py +++ b/src/serena/tools/memory_tools.py @@ -1,6 +1,6 @@ import json -from serena.tools import ReplaceRegexTool, Tool +from serena.tools import ReplaceContentTool, Tool class WriteMemoryTool(Tool): @@ -84,6 +84,6 @@ class EditMemoryTool(Tool): backreferences like \1, \2, etc. for groups matched by the regex. Insert new content verbatim, except for backslashes, which have to be escaped. """ - replace_regex_tool = self.agent.get_tool(ReplaceRegexTool) + replace_content_tool = self.agent.get_tool(ReplaceContentTool) rel_path = self.memories_manager.get_memory_file_path(memory_file_name).relative_to(self.get_project_root()) - return replace_regex_tool.replace_regex(str(rel_path), regex, repl, require_not_ignored=False) + return replace_content_tool.replace_content(str(rel_path), regex, repl, mode="regex", require_not_ignored=False) From b330a03a150bfc72cae0bfe6c62400c30366b95b Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 19 Nov 2025 15:04:25 +0100 Subject: [PATCH 2/6] Use new replace content functionality in EditMemoryTool Resolves #747 --- src/serena/tools/memory_tools.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/serena/tools/memory_tools.py b/src/serena/tools/memory_tools.py index e9280dc1..cc940257 100644 --- a/src/serena/tools/memory_tools.py +++ b/src/serena/tools/memory_tools.py @@ -1,4 +1,5 @@ import json +from typing import Literal from serena.tools import ReplaceContentTool, Tool @@ -70,20 +71,21 @@ class EditMemoryTool(Tool): def apply( self, memory_file_name: str, - regex: str, + needle: str, repl: str, + mode: Literal["literal", "regex"], ) -> str: r""" Replaces content matching a regular expression in a memory. :param memory_file_name: the name of the memory - :param regex: a Python-style regular expression (flags DOTALL and MULTILINE enabled, i.e. - '.' matches all characters, multi-line matching is enabled). - Apply the usual escaping as needed for reserved characters in Python-style regex. - :param repl: the string to replace the matched content with, which may contain - backreferences like \1, \2, etc. for groups matched by the regex. - Insert new content verbatim, except for backslashes, which have to be escaped. + :param needle: the string or regex pattern to search for. + If `mode` is "literal", this string will be matched exactly. + If `mode` is "regex", this string will be treated as a regular expression (syntax of Python's `re` module, + with flags DOTALL and MULTILINE enabled). + :param repl: the replacement string (verbatim). + :param mode: either "literal" or "regex", specifying how the `needle` parameter is to be interpreted. """ replace_content_tool = self.agent.get_tool(ReplaceContentTool) rel_path = self.memories_manager.get_memory_file_path(memory_file_name).relative_to(self.get_project_root()) - return replace_content_tool.replace_content(str(rel_path), regex, repl, mode="regex", require_not_ignored=False) + return replace_content_tool.replace_content(str(rel_path), needle, repl, mode=mode, require_not_ignored=False) From 2d2854d38ce537ae3b287d2c1cf50a3f2d5a74d0 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 19 Nov 2025 15:12:21 +0100 Subject: [PATCH 3/6] Simplify excessively long prompt on regex-based editing, fixing some incorrect statements --- src/serena/resources/config/modes/editing.yml | 81 ++----------------- 1 file changed, 7 insertions(+), 74 deletions(-) diff --git a/src/serena/resources/config/modes/editing.yml b/src/serena/resources/config/modes/editing.yml index ef62c2b7..ba468191 100644 --- a/src/serena/resources/config/modes/editing.yml +++ b/src/serena/resources/config/modes/editing.yml @@ -8,14 +8,12 @@ prompt: | When writing new code, think about where it belongs best. Don't generate new files if you don't plan on actually integrating them into the codebase, instead use the editing tools to insert the code directly into the existing files in that case. - You have two main approaches for editing code - editing by regex and editing by symbol. + You have two main approaches for editing code - editing at the symbol level and more file-based editing. The symbol-based approach is appropriate if you need to adjust an entire symbol, e.g. a method, a class, a function, etc. - But it is not appropriate if you need to adjust just a few lines of code within a symbol, for that you should - use the regex-based approach that is described below. + But it is not appropriate if you need to adjust just a few lines of code within a symbol. Let us first discuss the symbol-based approach. - Symbols are identified by their name path and relative file path, see the description of the `find_symbol` tool for more details - on how the `name_path` matches symbols. + Symbols are identified by their name path and relative file path (description of the `find_symbol` tool). You can get information about available symbols by using the `get_symbols_overview` tool for finding top-level symbols in a file, or by using `find_symbol` if you already know the symbol's name path. You generally try to read as little code as possible while still solving your task, meaning you only read the bodies when you need to, and after you have found the symbol you want to edit. @@ -35,76 +33,11 @@ prompt: | You can assume that all symbol editing tools are reliable, so you don't need to verify the results if the tool returns without error. {% if 'replace_content' in available_tools %} - Let us discuss the regex-based approach. - The regex-based approach is your primary tool for editing code whenever replacing or deleting a whole symbol would be a more expensive operation. - This is the case if you need to adjust just a few lines of code within a method, or a chunk that is much smaller than a whole symbol. - You use other tools to find the relevant content and - then use your knowledge of the codebase to write the regex, if you haven't collected enough information of this content yet. + The `replace_content` tool allows you to perform regex-based replacements within files. + This is your primary tool for editing code whenever replacing or deleting a whole symbol would be a more expensive operation, + e.g. if you need to adjust just a few lines of code within a method. You are extremely good at regex, so you never need to check whether the replacement produced the correct result. - In particular, you know what to escape and what not to escape, and you know how to use wildcards. - Also, the regex tool never adds any indentation (contrary to the symbolic editing tools), so you have to take care to add the correct indentation - when using it to insert code. - Moreover, the replacement tool will fail if it can't perform the desired replacement, and this is all the feedback you need. - Your overall goal for replacement operations is to use relatively short regexes, since I want you to minimize the number - of output tokens. For replacements of larger chunks of code, this means you intelligently make use of wildcards for the middle part - and of characteristic snippets for the before/after parts that uniquely identify the chunk. - - For small replacements, up to a single line, you follow the following rules: - - 1. If the snippet to be replaced is likely to be unique within the file, you perform the replacement by directly using the escaped version of the - original. - 2. If the snippet is probably not unique, and you want to replace all occurrences, you use the `allow_multiple_occurrences` flag. - 3. If the snippet is not unique, and you want to replace a specific occurrence, you make use of the code surrounding the snippet - to extend the regex with content before/after such that the regex will have exactly one match. - 4. You generally assume that a snippet is unique, knowing that the tool will return an error on multiple matches. You only read more file content - (for crafvarting a more specific regex) if such a failure unexpectedly occurs. - - Examples: - - 1 Small replacement - You have read code like - - ```python - ... - x = linear(x) - x = relu(x) - return x - ... - ``` - - and you want to replace `x = relu(x)` with `x = gelu(x)`. - You first try `replace_content` with the regex `x = relu\(x\)` and the replacement `x = gelu(x)`. - If this fails due to multiple matches, you will try `(linear\(x\)\s*)x = relu\(x\)(\s*return)` with the replacement `\1x = gelu(x)\2`. - - 2 Larger replacement - - You have read code like - - ```python - def my_func(): - ... - # a comment before the snippet - x = add_fifteen(x) - # beginning of long section within my_func - .... - # end of long section - call_subroutine(z) - call_second_subroutine(z) - ``` - and you want to replace the code starting with `x = add_fifteen(x)` until (including) `call_subroutine(z)`, but not `call_second_subroutine(z)`. - Initially, you assume that the the beginning and end of the chunk uniquely determine it within the file. - Therefore, you perform the replacement by using the regex `x = add_fifteen\(x\)\s*.*?call_subroutine\(z\)` - and the replacement being the new code you want to insert. - - If this fails due to multiple matches, you will try to extend the regex with the content before/after the snippet and match groups. - The matching regex becomes: - `(before the snippet\s*)x = add_fifteen\(x\)\s*.*?call_subroutine\(z\)` - and the replacement includes the group as (schematically): - `\1` - - Generally, I remind you that you rely on the regex tool with providing you the correct feedback, no need for more verification! - - IMPORTANT: REMEMBER TO USE WILDCARDS WHEN APPROPRIATE! I WILL BE VERY UNHAPPY IF YOU WRITE LONG REGEXES WITHOUT USING WILDCARDS INSTEAD! + In particular, you know how to use wildcards effectively in order to avoid specifying the full original text to be replaced! {% endif %} excluded_tools: - replace_lines From 9e1ce384facf6c16b42c32ba7a8aa463e791fa5e Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 19 Nov 2025 17:42:13 +0100 Subject: [PATCH 4/6] Handle legacy tool names in ToolSet for backward compatbility --- src/serena/config/serena_config.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index a5c3522f..e264e918 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -33,6 +33,7 @@ from serena.util.inspection import determine_programming_language_composition from solidlsp.ls_config import Language from ..analytics import RegisteredTokenCountEstimator +from ..tools import ReplaceContentTool from ..util.class_decorators import singleton if TYPE_CHECKING: @@ -70,6 +71,11 @@ class SerenaPaths: class ToolSet: + LEGACY_TOOL_NAME_MAPPING = {"replace_regex": ReplaceContentTool.get_name_from_cls()} + """ + maps legacy tool names to their new names for backward compatibility + """ + def __init__(self, tool_names: set[str]) -> None: self._tool_names = tool_names @@ -89,18 +95,28 @@ class ToolSet: """ from serena.tools import ToolRegistry + def get_updated_tool_name(tool_name: str) -> str: + """Retrieves the updated tool name if the provided tool name is deprecated, logging a warning.""" + if tool_name in self.LEGACY_TOOL_NAME_MAPPING: + new_tool_name = self.LEGACY_TOOL_NAME_MAPPING[tool_name] + log.warning("Tool name '%s' is deprecated, please use '%s' instead", tool_name, new_tool_name) + return new_tool_name + return tool_name + registry = ToolRegistry() tool_names = set(self._tool_names) for definition in tool_inclusion_definitions: included_tools = [] excluded_tools = [] for included_tool in definition.included_optional_tools: + included_tool = get_updated_tool_name(included_tool) if not registry.is_valid_tool_name(included_tool): raise ValueError(f"Invalid tool name '{included_tool}' provided for inclusion") if included_tool not in tool_names: tool_names.add(included_tool) included_tools.append(included_tool) for excluded_tool in definition.excluded_tools: + excluded_tool = get_updated_tool_name(excluded_tool) if not registry.is_valid_tool_name(excluded_tool): raise ValueError(f"Invalid tool name '{excluded_tool}' provided for exclusion") if excluded_tool in tool_names: From 98099e32d5cd34985025bf284786b462b1c11195 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 19 Nov 2025 17:49:38 +0100 Subject: [PATCH 5/6] Encourage use of regex mode more strongly --- src/serena/tools/file_tools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/serena/tools/file_tools.py b/src/serena/tools/file_tools.py index 96b58907..69e5c140 100644 --- a/src/serena/tools/file_tools.py +++ b/src/serena/tools/file_tools.py @@ -183,6 +183,7 @@ class ReplaceContentTool(Tool, ToolMarkerCanEdit): especially if it spans several lines. Note that you cannot make mistakes, because if the regex should match multiple occurrences while you disabled `allow_multiple_occurrences`, an error will be returned, and you can retry with a revised regex. + Therefore, using regex mode with suitable wildcards is usually the best choice! :param relative_path: the relative path to the file :param needle: the string or regex pattern to search for. From 2dcce39fa3ba87403ed73ab367c15b450f74b981 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 19 Nov 2025 18:02:50 +0100 Subject: [PATCH 6/6] Mention changes pertaining to ReplaceContentTool/ReplaceRegexTool in change log [skip ci] --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e79bc1b3..4ece5c3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ Status of the `main` branch. Changes prior to the next official version change w * Tools: * Added `RenameSymbolTool` for renaming symbols across the codebase (if LS supports this operation). + * Replaced `ReplaceRegexTool` with `ReplaceContentTool`, which supports both plain text and regex-based replacements + (and which requires no escaping in the replacement text, making it more robust) * Language support: