fix(tools): expand unmatched-group $!N backreferences to the empty string (#2070)

* fix(tools): expand unmatched-group $!N backreferences to the empty string

A $!N backreference in a regex-mode replacement template refers to the Nth
group of the search expression. When the group exists but did not participate
in the match (e.g. it sits inside an optional construct that was skipped), the
expansion emitted the literal template text instead of the empty string.

Observed in practice when an agent replaced 15 occurrences in an MQL header
with a template containing EA_INPUT$!1(...): the group was optional and never
participated, so every site came out containing the literal EA_INPUT$!1(...).

A reference to a group the expression does not define at all now raises a
clear ValueError instead of a raw IndexError - which also crashed
literal-mode replacements whose template contained $!N, since literal mode
compiles an escaped pattern without any groups.

* chore(ci): re-trigger test workflow

The previous run failed in native (macos-latest) on
test_cpp_basic.py::test_get_document_symbols[cpp_ccls] ("Expected 'main'
in document symbols, got: []"), a ccls language-server startup/indexing
flake unrelated to this PR's changes (text_utils.py and its tests only).

* fix(tools): pass literal-mode replacement through verbatim (no $!N expansion)

ContentReplacer ran the $!N backreference expansion on the replacement
template in both modes, so a literal-mode replacement whose template
contained a $!N sequence (e.g. documenting the convention itself in a
memory) failed with a backreference error instead of writing the text.
This contradicted the tools' documented contract ('the replacement string
(verbatim)' in literal mode) and the behavior of the dry-run path, where
MultiFileContentReplacer.find_occurrences already gated the expansion on
regex mode.

The expansion is now gated on the mode, mirroring the dry-run path; the
ambiguity validation still applies in both modes. The literal-mode test
that asserted the crash is inverted accordingly and the changelog entry
is corrected.
This commit is contained in:
davalillo authored and GitHub committed 2026-09-21 20:46:05 +02:00
1 parent 7ccffb6ee3
commit d80da2cdc6
3 files changed
+90 -12

No files matched your search

+56 -1
View File
@@ -3,7 +3,14 @@ from collections.abc import Callable
import pytest
from serena.util.file_proxy import FileCollection, FileProxy
from serena.util.text_utils import GlobMatcher, LineType, MultiFileContentReplacer, search_files, search_text
from serena.util.text_utils import (
ContentReplacer,
GlobMatcher,
LineType,
MultiFileContentReplacer,
search_files,
search_text,
)
class TestSearchText:
@@ -656,3 +663,51 @@ class TestMultiFileContentReplacer:
occ = replacer.find_occurrences([(path, content)], "old_pkg", "new_pkg")[0]
with pytest.raises(AssertionError):
replacer.apply_to_content("completely different content", [occ])
class TestBackreferenceExpansion:
"""$!N backreferences in regex-mode replacements refer to matched groups. A group that
exists but did not participate in the match (e.g. inside an optional construct that was
skipped) must expand to the empty string; a reference to a group that the search
expression does not define at all must fail with an error naming the problem instead of
a raw IndexError. Literal mode has no backreference expansion at all: the replacement is
used verbatim (observed in practice when an agent tried to document the $!N convention
itself and the literal-mode replacement crashed instead of writing the text).
"""
def test_unmatched_group_expands_to_empty_string(self):
replacer = ContentReplacer(mode="regex", allow_multiple_occurrences=False)
needle = r"EA_INPUT(?:\((\w*)\))?"
# the group participated and captured an empty string (empty parentheses)
assert replacer.replace("EA_INPUT()\n", needle, r"EA_INPUT$!1(...)") == "EA_INPUT(...)\n"
# the group did not participate at all (no parentheses)
assert replacer.replace("EA_INPUT\n", needle, r"EA_INPUT$!1(...)") == "EA_INPUT(...)\n"
def test_matched_group_expands_to_its_value(self):
replacer = ContentReplacer(mode="regex", allow_multiple_occurrences=False)
assert replacer.replace("id=alpha", r"id=(\w+)", r"[$!1]") == "[alpha]"
def test_nonexistent_group_reference_raises_clear_error(self):
replacer = ContentReplacer(mode="regex", allow_multiple_occurrences=False)
with pytest.raises(ValueError, match="does not exist"):
replacer.replace("id=alpha", r"id=(\w+)", r"[$!2]")
def test_literal_mode_repl_is_verbatim(self):
"""Literal mode has no groups at all and no backreference expansion: a replacement
containing $!N sequences is written as-is instead of failing with a backreference error.
"""
replacer = ContentReplacer(mode="literal", allow_multiple_occurrences=False)
assert replacer.replace("literal needle", "literal needle", "$!1 stuff $!2") == "$!1 stuff $!2"
def test_multi_file_replacer_expands_unmatched_group_to_empty_string(self):
replacer = MultiFileContentReplacer(mode="regex")
files = [("f.txt", "EA_INPUT\n")]
occurrences = replacer.find_occurrences(files, r"EA_INPUT(?:\((\w*)\))?", r"EA_INPUT$!1(...)")
assert [o.replacement for o in occurrences] == ["EA_INPUT(...)"]
def test_multi_file_replacer_nonexistent_group_reference_raises_clear_error(self):
replacer = MultiFileContentReplacer(mode="regex")
files = [("f.txt", "id=alpha\n")]
with pytest.raises(ValueError, match="does not exist"):
replacer.find_occurrences(files, r"id=(\w+)", r"[$!2]")