From b8a5eef5bc7c1e7f84cbd530524cc235f6157a44 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 14 May 2026 23:12:39 +0200 Subject: [PATCH] Change regex multi-line handling * Combine DOTALL and MULTILINE into a single flag for "multi-line mode", replacing the dotall parameter with multiline/regex_multiline * Apply the parameter in SearchForPatternTool * Remove the dotall flag from editing tools (ReplaceContentTool, EditMemoryTool) because edits should always use multi-line mode, but keep the multiline parameter as an option in ContentReplacer --- CHANGELOG.md | 9 ++----- src/serena/cli.py | 10 +------- src/serena/project.py | 12 ++++----- src/serena/tools/file_tools.py | 8 +++--- src/serena/tools/memory_tools.py | 9 ++----- src/serena/util/text_utils.py | 44 +++++++++++++++----------------- 6 files changed, 35 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9f67003..47046db5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,13 +24,8 @@ Status of the `main` branch. Changes prior to the next official version change w See the [CLI subcommands](https://oraios.github.io/serena/02-usage/045_memories.html#cli-subcommands). * Tools: - - `edit_memory`: New `dotall` parameter (default `False`) controlling whether ``.`` matches - newlines in regex mode. **Behavior change**: previously DOTALL was forced on; the new default - restricts ``.`` to a single line, which is safer for typical line-bounded memory edits and - avoids accidentally consuming large spans of content. Pass `dotall=True` for multi-line edits. - - `search_for_pattern`: New `dotall` parameter (default `True`, preserving existing behavior) - exposing the DOTALL flag, and the docstring was tightened. - - Delete `check_onboarding_performed` tool (instead extend project activation message), + - `search_for_pattern`: Add parameter `multiline` + - Delete `check_onboarding_performed` tool (instead extend project activation message) # v1.3.0 (2026-05-11) diff --git a/src/serena/cli.py b/src/serena/cli.py index 27022662..5f87a36b 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -1267,7 +1267,7 @@ class MemoryCommands(AutoRegisteringGroup): type=click.Choice(["literal", "regex"]), default="literal", show_default=True, - help="Treat --needle as literal text or as a Python regex (MULTILINE flag is always on; DOTALL via --dotall).", + help="Treat --needle as literal text or as a Python regex (MULTILINE and DOTALL flags enabled).", ) @click.option( "--allow-multiple-occurrences", @@ -1275,12 +1275,6 @@ class MemoryCommands(AutoRegisteringGroup): default=False, help="Permit and apply multiple matches; without this, multiple matches raise an error.", ) - @click.option( - "--dotall", - is_flag=True, - default=False, - help="Regex mode only: make `.` match newlines as well, enabling multi-line patterns. Use with care.", - ) def edit( memory_name: str, project: str, @@ -1288,7 +1282,6 @@ class MemoryCommands(AutoRegisteringGroup): repl: str, mode: Literal["literal", "regex"], allow_multiple_occurrences: bool, - dotall: bool, ) -> None: manager = MemoryCommands._load_memories_manager(project) click.echo( @@ -1299,7 +1292,6 @@ class MemoryCommands(AutoRegisteringGroup): mode, allow_multiple_occurrences, is_tool_context=False, - dotall=dotall, ) ) diff --git a/src/serena/project.py b/src/serena/project.py index 5ba84ca6..8fc59817 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -358,7 +358,7 @@ class MemoriesManager: mode: Literal["literal", "regex"], allow_multiple_occurrences: bool, is_tool_context: bool, - dotall: bool = True, + regex_multiline: bool = True, ) -> str: """ Edit a memory by replacing content matching a pattern. @@ -369,7 +369,7 @@ class MemoriesManager: :param mode: "literal" or "regex" :param allow_multiple_occurrences: :param is_tool_context: whether the call originates from a tool invocation (affects write-access checks) - :param dotall: whether to compile the regex with the DOTALL flag (``.`` matches newlines). Only relevant in regex mode. + :param regex_multiline: whether to apply multi-line regex matching, enabling the flags re.DOTALL and re.MULTILINE """ name = self._prepare_name(name) self._check_not_ignored(name) @@ -379,7 +379,7 @@ class MemoriesManager: raise FileNotFoundError(f"Memory {name} not found.") with open(memory_file_path, encoding=self._encoding) as f: original_content = f.read() - replacer = ContentReplacer(mode=mode, allow_multiple_occurrences=allow_multiple_occurrences, dotall=dotall) + replacer = ContentReplacer(mode=mode, allow_multiple_occurrences=allow_multiple_occurrences, regex_multiline=regex_multiline) updated_content = replacer.replace(original_content, needle, repl) with open(memory_file_path, "w", encoding=self._encoding) as f: f.write(updated_content) @@ -738,7 +738,7 @@ class Project(ToStringMixin): context_lines_after: int = 0, paths_include_glob: str | None = None, paths_exclude_glob: str | None = None, - dotall: bool = True, + multiline: bool = True, ) -> list[MatchedConsecutiveLines]: """ Search for a pattern across all (non-ignored) source files @@ -749,7 +749,7 @@ class Project(ToStringMixin): :param context_lines_after: Number of lines of context to include after each match :param paths_include_glob: Glob pattern to filter which files to include in the search :param paths_exclude_glob: Glob pattern to filter which files to exclude from the search. Takes precedence over paths_include_glob. - :param dotall: Whether to compile the regex with the DOTALL flag (``.`` matches newlines). + :param multiline: Whether to compile the regex with the DOTALL flag (``.`` matches newlines). :return: List of matched consecutive lines with context """ relative_file_paths = self.gather_source_files(relative_path=relative_path) @@ -762,7 +762,7 @@ class Project(ToStringMixin): context_lines_after=context_lines_after, paths_include_glob=paths_include_glob, paths_exclude_glob=paths_exclude_glob, - dotall=dotall, + multiline=multiline, ) def retrieve_content_around_line( diff --git a/src/serena/tools/file_tools.py b/src/serena/tools/file_tools.py index 83aa837e..830e0bfe 100644 --- a/src/serena/tools/file_tools.py +++ b/src/serena/tools/file_tools.py @@ -332,7 +332,7 @@ class SearchForPatternTool(Tool): paths_exclude_glob: str = "", relative_path: str = "", restrict_search_to_code_files: bool = False, - dotall: bool = True, + multiline: bool = True, max_answer_chars: int = -1, ) -> str: """ @@ -347,7 +347,7 @@ class SearchForPatternTool(Tool): :param relative_path: restricts the search to this file or subdirectory of the project root. Must exist. :param restrict_search_to_code_files: whether to search only files containing analyzable code symbols (useful when looking for class/method definitions); otherwise also search non-code files. - :param dotall: whether to compile the regex with the DOTALL flag (``.`` matches newlines, enabling multi-line matches). + :param multiline: whether to apply multi-line matching, enabling the flags re.DOTALL and re.MULTILINE :param max_answer_chars: if the output exceeds this many characters, a progressively shortened summary is returned instead. ``-1`` uses the configured default. :return: A mapping from file paths to matched consecutive lines (0-based line numbers). @@ -368,7 +368,7 @@ class SearchForPatternTool(Tool): context_lines_after=context_lines_after, paths_include_glob=paths_include_glob.strip(), paths_exclude_glob=paths_exclude_glob.strip(), - dotall=dotall, + multiline=multiline, ) else: if os.path.isfile(abs_path): @@ -392,7 +392,7 @@ class SearchForPatternTool(Tool): root_path=self.get_project_root(), paths_include_glob=paths_include_glob, paths_exclude_glob=paths_exclude_glob, - dotall=dotall, + multiline=multiline, ) # group matches by file diff --git a/src/serena/tools/memory_tools.py b/src/serena/tools/memory_tools.py index a1349c79..f7be84d0 100644 --- a/src/serena/tools/memory_tools.py +++ b/src/serena/tools/memory_tools.py @@ -103,7 +103,6 @@ class EditMemoryTool(Tool, ToolMarkerCanEdit): repl: str, mode: Literal["literal", "regex"], allow_multiple_occurrences: bool = False, - dotall: bool = False, ) -> str: r""" Replace content matching a regular expression in a memory. @@ -112,16 +111,12 @@ class EditMemoryTool(Tool, ToolMarkerCanEdit): :param needle: the string or regex pattern to search for. In regex mode, be careful to not replace too much! If `mode` is "literal", this string will be matched exactly. If `mode` is "regex", this string will be treated as a regular expression (syntax of Python's `re` module, - with the MULTILINE flag enabled and DOTALL controlled by the `dotall` parameter). + with the MULTILINE and DOTALL flags enabled). :param repl: the replacement string (verbatim). :param mode: either "literal" or "regex", specifying how the `needle` parameter is to be interpreted. :param allow_multiple_occurrences: whether to allow matching and replacing multiple occurrences. If false and multiple occurrences are found, an error will be returned. - :param dotall: whether to compile the regex with the DOTALL flag, so that ``.`` matches newlines. - Only relevant in regex mode. Defaults to False, restricting ``.`` to a single line, which is the safer - choice for typical line-bounded memory edits and avoids accidentally consuming entire blocks of content. - Set to True deliberately when you intend a multi-line replacement (alternatively, use ``[\s\S]*?``). """ return self.memories_manager.edit_memory( - memory_name, needle, repl, mode, allow_multiple_occurrences, is_tool_context=True, dotall=dotall + memory_name, needle, repl, mode, allow_multiple_occurrences, is_tool_context=True, regex_multiline=True ) diff --git a/src/serena/util/text_utils.py b/src/serena/util/text_utils.py index ddadf549..16bee9d9 100644 --- a/src/serena/util/text_utils.py +++ b/src/serena/util/text_utils.py @@ -145,7 +145,7 @@ def search_text( context_lines_before: int = 0, context_lines_after: int = 0, is_glob: bool = False, - dotall: bool = True, + multiline: bool = True, ) -> list[MatchedConsecutiveLines]: """ Search for a pattern in text content. Supports both regex and glob-like patterns. @@ -160,13 +160,9 @@ def search_text( :param context_lines_after: Number of context lines to include after matches :param is_glob: If True, pattern is treated as a glob-like pattern (e.g., "*.py", "test_??.py") and will be converted to regex internally - :param dotall: Whether to compile the regex with the DOTALL flag (``.`` matches newlines). - Only takes effect when `allow_multiline_match` is True (the line-by-line branch never spans newlines). - + :param multiline: whether to apply multi-line matching, enabling the flags re.DOTALL and re.MULTILINE :return: List of `TextSearchMatch` objects - :raises: ValueError if the pattern is not valid - """ if source_file_path and content is None: with open(source_file_path) as f: @@ -184,7 +180,8 @@ def search_text( pattern = glob_to_regex(pattern) if allow_multiline_match: # For multiline matches, optionally use DOTALL so '.' matches newlines - compiled_pattern = re.compile(pattern, re.DOTALL if dotall else 0) + flags = (re.MULTILINE | re.DOTALL) if multiline else 0 + compiled_pattern = re.compile(pattern, flags) # Search across the entire content as a single string for match in compiled_pattern.finditer(content): start_pos = match.start() @@ -330,22 +327,22 @@ def search_files( context_lines_after: int = 0, paths_include_glob: str | None = None, paths_exclude_glob: str | None = None, - dotall: bool = True, + multiline: bool = True, ) -> list[MatchedConsecutiveLines]: """ Search for a pattern in a list of files. - :param relative_file_paths: List of relative file paths in which to search - :param pattern: Pattern to search for - :param root_path: Root path to resolve relative paths against (by default, current working directory). - :param file_reader: Function to read a file, by default will just use os.open. + :param relative_file_paths: list of relative file paths in which to search + :param pattern: pattern to search for + :param root_path: root path to resolve relative paths against (by default, current working directory). + :param file_reader: function to read a file, by default will just use os.open. All files that can't be read by it will be skipped. - :param context_lines_before: Number of context lines to include before matches - :param context_lines_after: Number of context lines to include after matches - :param paths_include_glob: Optional glob pattern to include files from the list - :param paths_exclude_glob: Optional glob pattern to exclude files from the list - :param dotall: Whether to compile the regex with the DOTALL flag (``.`` matches newlines). - :return: List of MatchedConsecutiveLines objects + :param context_lines_before: number of context lines to include before matches + :param context_lines_after: number of context lines to include after matches + :param paths_include_glob: optional glob pattern to include files from the list + :param paths_exclude_glob: optional glob pattern to exclude files from the list + :param multiline: whether to apply multi-line matching, enabling the flags re.DOTALL and re.MULTILINE (default: True) + :return: list of MatchedConsecutiveLines objects """ # Pre-filter paths (done sequentially to avoid overhead) # Use proper glob matching instead of gitignore patterns @@ -380,7 +377,7 @@ def search_files( allow_multiline_match=True, context_lines_before=context_lines_before, context_lines_after=context_lines_after, - dotall=dotall, + multiline=multiline, ) if len(search_results) > 0: log.debug(f"Found {len(search_results)} matches in {path}") @@ -436,19 +433,18 @@ class ContentReplacer: provides dual modes for maximum flexibility. """ - def __init__(self, mode: Literal["literal", "regex"], allow_multiple_occurrences: bool, dotall: bool = True): + def __init__(self, mode: Literal["literal", "regex"], allow_multiple_occurrences: bool, regex_multiline: bool = True): """ :param mode: the mode indicating whether to the needle in replacements corresponds to a regular expression (mode "regex") or to a literal string (mode "literal") :param allow_multiple_occurrences: whether it is allowed that the search expression matches multiple occurrences. If False, an error will be raised if more than one match is found. - :param dotall: whether to compile the regex with the DOTALL flag, which makes ``.`` match newlines. - Only relevant in regex mode. The MULTILINE flag is always enabled (so ``^`` and ``$`` match line boundaries). + :param regex_multiline: whether to apply multi-line regex matching, enabling the flags re.DOTALL and re.MULTILINE """ self.mode = mode self.allow_multiple_occurrences = allow_multiple_occurrences - self.dotall = dotall + self.regex_multiline = regex_multiline @staticmethod def _create_replacement_function(regex_pattern: str, repl_template: str, regex_flags: int) -> Callable[[re.Match], str]: @@ -514,7 +510,7 @@ class ContentReplacer: else: raise ValueError(f"Invalid mode: '{self.mode}', expected 'literal' or 'regex'.") - regex_flags = re.MULTILINE | (re.DOTALL if self.dotall else 0) + regex_flags = (re.MULTILINE | re.DOTALL) if self.regex_multiline else 0 # create replacement function with validation and backreference handling repl_fn = self._create_replacement_function(regex, repl, regex_flags=regex_flags)