From 15ab4e7747c46ebf6594baaa6610b098c26ae9ca Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 24 Feb 2026 11:28:26 +0100 Subject: [PATCH 1/2] Move test_utils to util package --- .../jetbrains/jetbrains_plugin_client.py | 2 +- src/serena/project.py | 2 +- src/serena/tools/file_tools.py | 69 ++----------------- src/serena/{ => util}/text_utils.py | 2 +- src/solidlsp/ls.py | 2 +- test/serena/test_text_utils.py | 6 +- test/solidlsp/python/test_python_basic.py | 2 +- test/solidlsp/swift/test_swift_basic.py | 2 +- 8 files changed, 13 insertions(+), 74 deletions(-) rename src/serena/{ => util}/text_utils.py (99%) diff --git a/src/serena/jetbrains/jetbrains_plugin_client.py b/src/serena/jetbrains/jetbrains_plugin_client.py index 25f02bea..647f4bda 100644 --- a/src/serena/jetbrains/jetbrains_plugin_client.py +++ b/src/serena/jetbrains/jetbrains_plugin_client.py @@ -15,7 +15,7 @@ from sensai.util.string import ToStringMixin import serena.jetbrains.jetbrains_types as jb from serena.jetbrains.jetbrains_types import PluginStatusDTO from serena.project import Project -from serena.text_utils import render_html +from serena.util.text_utils import render_html from serena.util.version import Version T = TypeVar("T") diff --git a/src/serena/project.py b/src/serena/project.py index e66a8732..59b89ec4 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -16,8 +16,8 @@ from serena.config.serena_config import ( ) from serena.constants import SERENA_FILE_ENCODING, SERENA_MANAGED_DIR_NAME from serena.ls_manager import LanguageServerFactory, LanguageServerManager -from serena.text_utils import MatchedConsecutiveLines, search_files from serena.util.file_system import GitignoreParser, match_path +from serena.util.text_utils import MatchedConsecutiveLines, search_files from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language from solidlsp.ls_utils import FileUtils diff --git a/src/serena/tools/file_tools.py b/src/serena/tools/file_tools.py index 66513f9a..50ad0099 100644 --- a/src/serena/tools/file_tools.py +++ b/src/serena/tools/file_tools.py @@ -14,9 +14,10 @@ from fnmatch import fnmatch from pathlib import Path from typing import Literal -from serena.text_utils import search_files from serena.tools import SUCCESS_RESULT, EditedFileContext, Tool, ToolMarkerCanEdit, ToolMarkerOptional from serena.util.file_system import scan_directory +from serena.util.text_utils import search_files +from temp.temp_temp import ContentReplacer class ReadFileTool(Tool): @@ -203,46 +204,6 @@ class ReplaceContentTool(Tool, ToolMarkerCanEdit): relative_path, needle, repl, mode=mode, allow_multiple_occurrences=allow_multiple_occurrences, require_not_ignored=True ) - @staticmethod - def _create_replacement_function(regex_pattern: str, repl_template: str, regex_flags: int) -> Callable[[re.Match], str]: - """ - Creates a replacement function that validates for ambiguity and handles backreferences. - - :param regex_pattern: The regex pattern being used for matching - :param repl_template: The replacement template with $!1, $!2, etc. for backreferences - :param regex_flags: The flags to use when searching (e.g., re.DOTALL | re.MULTILINE) - :return: A function suitable for use with re.sub() or re.subn() - """ - - def validate_and_replace(match: re.Match) -> str: - matched_text = match.group(0) - - # For multi-line match, check if the same pattern matches again within the already-matched text, - # rendering the match ambiguous. Typical pattern in the code: - # - # When matching - # .*? - # this will match the entire span above, while only the suffix may have been intended. - # (See test case for a practical example.) - # To detect this, we check if the same pattern matches again within the matched text, - if "\n" in matched_text and re.search(regex_pattern, matched_text[1:], flags=regex_flags): - raise ValueError( - "Match is ambiguous: the search pattern matches multiple overlapping occurrences. " - "Please revise the search pattern to be more specific to avoid ambiguity, " - "e.g. by matching specific context after the match, or try using the literal mode." - ) - - # Handle backreferences: replace $!1, $!2, etc. with actual matched groups - def expand_backreference(m: re.Match) -> str: - group_num = int(m.group(1)) - group_value = match.group(group_num) - return group_value if group_value is not None else m.group(0) - - result = re.sub(r"\$!(\d+)", expand_backreference, repl_template) - return result - - return validate_and_replace - def replace_content( self, relative_path: str, @@ -259,31 +220,9 @@ class ReplaceContentTool(Tool, ToolMarkerCanEdit): self.project.validate_relative_path(relative_path, require_not_ignored=require_not_ignored) with EditedFileContext(relative_path, self.create_code_editor()) as context: original_content = context.get_original_content() - - if mode == "literal": - regex = re.escape(needle) - elif mode == "regex": - regex = needle - else: - raise ValueError(f"Invalid mode: '{mode}', expected 'literal' or 'regex'.") - - regex_flags = re.DOTALL | re.MULTILINE - - # create replacement function with validation and backreference handling - repl_fn = self._create_replacement_function(regex, repl, regex_flags=regex_flags) - - # perform replacement - updated_content, n = re.subn(regex, repl_fn, original_content, flags=regex_flags) - - if n == 0: - raise ValueError(f"Error: No matches of search expression found in file '{relative_path}'.") - if not allow_multiple_occurrences and n > 1: - raise ValueError( - f"Expression matches {n} occurrences in file '{relative_path}'. " - "Please revise the expression to be more specific or enable allow_multiple_occurrences if this is expected." - ) + replacer = ContentReplacer(mode=mode, allow_multiple_occurrences=allow_multiple_occurrences) + updated_content = replacer.replace(original_content, needle, repl) context.set_updated_content(updated_content) - return SUCCESS_RESULT diff --git a/src/serena/text_utils.py b/src/serena/util/text_utils.py similarity index 99% rename from src/serena/text_utils.py rename to src/serena/util/text_utils.py index c36dce78..b42c1e90 100644 --- a/src/serena/text_utils.py +++ b/src/serena/util/text_utils.py @@ -123,7 +123,7 @@ def glob_to_regex(glob_pat: str) -> str: if ch == "*": regex_parts.append(".*") elif ch == "?": - regex_parts.append(".") + regex_parts.append("..") elif ch == "\\": i += 1 if i < len(glob_pat): diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index 1b53045a..e6f247c5 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -20,8 +20,8 @@ import pathspec from sensai.util.pickle import getstate, load_pickle from sensai.util.string import ToStringMixin -from serena.text_utils import MatchedConsecutiveLines from serena.util.file_system import match_path +from serena.util.text_utils import MatchedConsecutiveLines from solidlsp import ls_types from solidlsp.ls_config import Language, LanguageServerConfig from solidlsp.ls_exceptions import SolidLSPException diff --git a/test/serena/test_text_utils.py b/test/serena/test_text_utils.py index d3701ad9..0b8dd39d 100644 --- a/test/serena/test_text_utils.py +++ b/test/serena/test_text_utils.py @@ -2,7 +2,7 @@ import re import pytest -from serena.text_utils import LineType, search_files, search_text +from serena.util.text_utils import LineType, search_files, search_text class TestSearchText: @@ -545,7 +545,7 @@ class TestGlobMatch: ) def test_glob_match(self, pattern, path, expected): """Test glob_match function with various patterns.""" - from src.serena.text_utils import glob_match + from serena.util.text_utils import glob_match assert glob_match(pattern, path) == expected @@ -572,6 +572,6 @@ class TestExpandBraces: ) def test_expand_braces(self, pattern, expected): """Test brace expansion for glob patterns.""" - from serena.text_utils import expand_braces + from serena.util.text_utils import expand_braces assert sorted(expand_braces(pattern)) == sorted(expected) diff --git a/test/solidlsp/python/test_python_basic.py b/test/solidlsp/python/test_python_basic.py index 4b60995b..faeca67a 100644 --- a/test/solidlsp/python/test_python_basic.py +++ b/test/solidlsp/python/test_python_basic.py @@ -10,7 +10,7 @@ import os import pytest from serena.project import Project -from serena.text_utils import LineType +from serena.util.text_utils import LineType from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language diff --git a/test/solidlsp/swift/test_swift_basic.py b/test/solidlsp/swift/test_swift_basic.py index d3934f5f..bd0852d5 100644 --- a/test/solidlsp/swift/test_swift_basic.py +++ b/test/solidlsp/swift/test_swift_basic.py @@ -11,7 +11,7 @@ import platform import pytest from serena.project import Project -from serena.text_utils import LineType +from serena.util.text_utils import LineType from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language from test.conftest import is_ci From e38d8de21b782a1a1423da1f4b0737da6a7e5bc9 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 24 Feb 2026 11:33:21 +0100 Subject: [PATCH 2/2] Factor out ContentReplacer from ReplaceContentTool --- src/serena/tools/file_tools.py | 5 +- src/serena/util/text_utils.py | 101 ++++++++++++++++++++++++++++++++- 2 files changed, 101 insertions(+), 5 deletions(-) diff --git a/src/serena/tools/file_tools.py b/src/serena/tools/file_tools.py index 50ad0099..591f4f01 100644 --- a/src/serena/tools/file_tools.py +++ b/src/serena/tools/file_tools.py @@ -7,17 +7,14 @@ File and file system-related tools, specifically for """ import os -import re from collections import defaultdict -from collections.abc import Callable from fnmatch import fnmatch from pathlib import Path from typing import Literal from serena.tools import SUCCESS_RESULT, EditedFileContext, Tool, ToolMarkerCanEdit, ToolMarkerOptional from serena.util.file_system import scan_directory -from serena.util.text_utils import search_files -from temp.temp_temp import ContentReplacer +from serena.util.text_utils import ContentReplacer, search_files class ReadFileTool(Tool): diff --git a/src/serena/util/text_utils.py b/src/serena/util/text_utils.py index b42c1e90..e3c63e77 100644 --- a/src/serena/util/text_utils.py +++ b/src/serena/util/text_utils.py @@ -5,7 +5,7 @@ import re from collections.abc import Callable from dataclasses import dataclass, field from enum import StrEnum -from typing import Any, Self +from typing import Any, Literal, Self from bs4 import BeautifulSoup from joblib import Parallel, delayed @@ -424,3 +424,102 @@ def render_html(html: str) -> str: text = text.replace("\xa0", " ") return text.strip() + + +class ContentReplacer: + """ + This is an LLM-optimised content replacer, which elegantly circumvents escaping and which + provides dual modes for maximum flexibility. + """ + + def __init__(self, mode: Literal["literal", "regex"], allow_multiple_occurrences: bool): + """ + + :param mode: the mode indicating whether to the needle in replacements corresponds to a regular expression + (mode "regex") or to a literal string (mode "literal") + :param allow_multiple_occurrences: whether it is allowed that the search expression matches multiple occurrences. + If False, an error will be raised if more than one match is found. + """ + self.mode = mode + self.allow_multiple_occurrences = allow_multiple_occurrences + + @staticmethod + def _create_replacement_function(regex_pattern: str, repl_template: str, regex_flags: int) -> Callable[[re.Match], str]: + """ + Creates a replacement function that validates for ambiguity and handles backreferences. + + :param regex_pattern: The regex pattern being used for matching + :param repl_template: The replacement template with $!1, $!2, etc. for backreferences + :param regex_flags: The flags to use when searching (e.g., re.DOTALL | re.MULTILINE) + :return: A function suitable for use with re.sub() or re.subn() + """ + + def validate_and_replace(match: re.Match) -> str: + matched_text = match.group(0) + + # For multi-line match, check if the same pattern matches again within the already-matched text, + # rendering the match ambiguous. Typical pattern in the code: + # + # When matching + # .*? + # this will match the entire span above, while only the suffix may have been intended. + # (See test case for a practical example.) + # To detect this, we check if the same pattern matches again within the matched text, + if "\n" in matched_text and re.search(regex_pattern, matched_text[1:], flags=regex_flags): + raise ValueError( + "Match is ambiguous: the search pattern matches multiple overlapping occurrences. " + "Please revise the search pattern to be more specific to avoid ambiguity, " + "e.g. by matching specific context after the match, or try using the literal mode." + ) + + # Handle backreferences: replace $!1, $!2, etc. with actual matched groups + def expand_backreference(m: re.Match) -> str: + group_num = int(m.group(1)) + group_value = match.group(group_num) + return group_value if group_value is not None else m.group(0) + + result = re.sub(r"\$!(\d+)", expand_backreference, repl_template) + return result + + return validate_and_replace + + def replace( + self, + content: str, + needle: str, + repl: str, + ) -> str: + """ + Performs the replacement. + + Raises ValueError if no match is found, or if multiple matches are found while allow_multiple_occurrences is False. + + :param content: the content in which to perform the replacement + :param needle: the search expression, which is either a literal string or a regular expression, depending on the mode + :param repl: the replacement string, which, in regex mode, may contain backreferences in the form of $!1, $!2, etc. to + refer to matched groups in the search expression + :return: the updated content after performing the replacement + """ + if self.mode == "literal": + regex = re.escape(needle) + elif self.mode == "regex": + regex = needle + else: + raise ValueError(f"Invalid mode: '{self.mode}', expected 'literal' or 'regex'.") + + regex_flags = re.DOTALL | re.MULTILINE + + # create replacement function with validation and backreference handling + repl_fn = self._create_replacement_function(regex, repl, regex_flags=regex_flags) + + # perform replacement + updated_content, n = re.subn(regex, repl_fn, content, flags=regex_flags) + + if n == 0: + raise ValueError("Error: No matches of search expression found.") + if not self.allow_multiple_occurrences and n > 1: + raise ValueError( + f"Expression matches {n} occurrences. " + "Please revise the expression to be more specific or enable allow_multiple_occurrences if this is expected." + ) + return updated_content