diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f7b9b04..e214da84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Status of the `main` branch. Changes prior to the next official version change w - Add Grok Build support (context `grok`, setup CLI, hooks) - The `languages` key in project configurations was changed to `language_servers` to better reflect the actual semantics (configurations are automatically migrated) + - Fix: glob matching bare `*` and `?` in non-`**` patterns matched across `/`, contradicting documented behaviour #1732 * Language Servers: - Allow language server priorities to be configured in `serena_config.yml` (for auto-detection during diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index 063cc658..9a169962 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -33,7 +33,7 @@ from serena.constants import ( SERENA_MANAGED_DIR_NAME, ) from serena.util.inspection import compute_language_server_support_composition -from serena.util.text_utils import glob_match +from serena.util.text_utils import GlobMatcher from serena.util.yaml import YamlCommentNormalisation, load_yaml, normalise_yaml_comments, save_yaml, transfer_yaml_comments from solidlsp.ls_config import LanguageServerId @@ -1437,7 +1437,7 @@ class SerenaConfig(SharedConfig, ModeSelectionDefinitionWithBaseModes): """ project_root_str = str(project_root) for pattern in self.trusted_project_path_patterns: - if glob_match(pattern, project_root_str): + if GlobMatcher(pattern).matches(project_root_str): return True return False diff --git a/src/serena/tools/file_tools.py b/src/serena/tools/file_tools.py index 3cc55d82..bd52f5db 100644 --- a/src/serena/tools/file_tools.py +++ b/src/serena/tools/file_tools.py @@ -16,10 +16,9 @@ from serena.tools import SUCCESS_RESULT, EditedFileContext, EditingToolWithDiagn from serena.util.file_system import scan_directory from serena.util.text_utils import ( ContentReplacer, + GlobMatcher, MultiFileContentReplacer, ReplacementOccurrence, - expand_braces, - glob_match, ) from solidlsp.ls_utils import TextUtils @@ -350,13 +349,13 @@ class ReplaceInFilesTool(EditingToolWithDiagnostics): is_ignored_file=self.project.is_ignored_path, relative_to=self.get_project_root(), ) - include_patterns = expand_braces(paths_include_glob.strip()) if paths_include_glob.strip() else None - exclude_patterns = expand_braces(paths_exclude_glob.strip()) if paths_exclude_glob.strip() else None + include_glob_matcher = GlobMatcher(paths_include_glob.strip()) if paths_include_glob.strip() else None + exclude_glob_matcher = GlobMatcher(paths_exclude_glob.strip()) if paths_exclude_glob.strip() else None files: list[tuple[str, str]] = [] for path in sorted(rel_paths): - if include_patterns and not any(glob_match(p, path) for p in include_patterns): + if include_glob_matcher and not include_glob_matcher.matches(path): continue - if exclude_patterns and any(glob_match(p, path) for p in exclude_patterns): + if exclude_glob_matcher and exclude_glob_matcher.matches(path): continue try: files.append((path, self.project.read_file(path))) diff --git a/src/serena/util/text_utils.py b/src/serena/util/text_utils.py index 4f8f10cc..8ab6f943 100644 --- a/src/serena/util/text_utils.py +++ b/src/serena/util/text_utils.py @@ -1,4 +1,3 @@ -import fnmatch import hashlib import logging import re @@ -9,6 +8,7 @@ from typing import Any, Literal, Self from bs4 import BeautifulSoup from joblib import Parallel, delayed +from sensai.util.string import ToStringMixin from serena.util.file_proxy import FileCollection, FileProxy from solidlsp.ls_utils import TextUtils @@ -116,34 +116,12 @@ class MatchedConsecutiveLines: return cls(lines=text_lines, source_file_path=source_file_path) -def glob_to_regex(glob_pat: str) -> str: - regex_parts: list[str] = [] - i = 0 - while i < len(glob_pat): - ch = glob_pat[i] - if ch == "*": - regex_parts.append(".*") - elif ch == "?": - regex_parts.append(".") - elif ch == "\\": - i += 1 - if i < len(glob_pat): - regex_parts.append(re.escape(glob_pat[i])) - else: - regex_parts.append("\\") - else: - regex_parts.append(re.escape(ch)) - i += 1 - return "".join(regex_parts) - - def search_text( pattern: str, content: str | None = None, source_file_path: str | None = None, context_lines_before: int = 0, context_lines_after: int = 0, - is_glob: bool = False, multiline: bool = True, ) -> list[MatchedConsecutiveLines]: """ @@ -155,8 +133,6 @@ def search_text( this has to be passed and the file will be read. :param context_lines_before: Number of context lines to include before matches :param context_lines_after: Number of context lines to include after matches - :param is_glob: If True, pattern is treated as a glob-like pattern (e.g., "*.py", "test_??.py") - and will be converted to regex internally :param multiline: whether to apply multi-line matching, enabling the flags re.DOTALL and re.MULTILINE :return: List of `TextSearchMatch` objects :raises: ValueError if the pattern is not valid @@ -172,10 +148,6 @@ def search_text( lines = TextUtils.split_lines(content) total_lines = len(lines) - # Convert pattern to a compiled regex if it's a string - if is_glob: - pattern = glob_to_regex(pattern) - # For multiline matches, optionally use DOTALL so '.' matches newlines flags = (re.MULTILINE | re.DOTALL) if multiline else 0 compiled_pattern = re.compile(pattern, flags) @@ -213,36 +185,9 @@ def search_text( return matches -def expand_braces(pattern: str) -> list[str]: +class GlobMatcher(ToStringMixin): """ - Expands brace patterns in a glob string. - For example, "**/*.{js,jsx,ts,tsx}" becomes ["**/*.js", "**/*.jsx", "**/*.ts", "**/*.tsx"]. - Handles multiple brace sets as well. - """ - patterns = [pattern] - while any("{" in p or "}" in p for p in patterns): - new_patterns = [] - for p in patterns: - match = re.search(r"\{([^{}]*)\}", p) - if match: - options_expr = match.group(1) - if not options_expr: - raise ValueError(f"Invalid glob brace expression in {pattern!r}: empty braces are not allowed") - - prefix = p[: match.start()] - suffix = p[match.end() :] - options = options_expr.split(",") - for option in options: - new_patterns.append(f"{prefix}{option}{suffix}") - else: - raise ValueError(f"Invalid glob brace expression in {pattern!r}: unmatched brace") - patterns = new_patterns - return patterns - - -def glob_match(pattern: str, path: str) -> bool: - """ - Match a file path against a glob pattern. + Supports matching a file path against a glob pattern. Supports standard glob patterns: - * matches any number of characters except / @@ -252,53 +197,100 @@ def glob_match(pattern: str, path: str) -> bool: Supports brace expansion: - {a,b,c} expands to multiple patterns (including nesting) - - Unsupported patterns: - - Bash extended glob features are unavailable in Python's fnmatch - - Extended globs like !(), ?(), +(), *(), @() are not supported - - :param pattern: Glob pattern (e.g., 'src/**/*.py', '**agent.py') - :param path: File path to match against - :return: True if path matches pattern """ - pattern = pattern.replace("\\", "/") # Normalize backslashes to forward slashes - path = path.replace("\\", "/") # Normalize path backslashes to forward slashes - # Handle ** patterns that should match zero or more directories - if "**" in pattern: - # Method 1: Standard fnmatch (matches one or more directories) - regex1 = fnmatch.translate(pattern) - if re.match(regex1, path): - return True - - # Method 2: Handle zero-directory case by removing /** entirely - # Convert "src/**/test.py" to "src/test.py" - if "/**/" in pattern: - zero_dir_pattern = pattern.replace("/**/", "/") - regex2 = fnmatch.translate(zero_dir_pattern) - if re.match(regex2, path): - return True - - # Method 3: Handle leading ** case by removing **/ - # Convert "**/test.py" to "test.py" - if pattern.startswith("**/"): - zero_dir_pattern = pattern[3:] # Remove "**/" - regex3 = fnmatch.translate(zero_dir_pattern) - if re.match(regex3, path): - return True - - return False - else: - # Simple pattern without **, use fnmatch directly - return fnmatch.fnmatch(path, pattern) - - -class GlobMatcher: def __init__(self, expr: str): - self._patterns = expand_braces(expr) + """ + :param expr: a glob pattern which may make use of brace expansion (e.g., "src/**/*.{js,jsx,ts,tsx}") + """ + expr = expr.replace("\\", "/") # normalise backslashes to forward slashes + self._glob_expr = expr + self._glob_patterns = self._expand_braces(expr) + self._regex_patterns = [re.compile(self._translate_glob_to_regex(p)) for p in self._glob_patterns] + + def _tostring_includes(self) -> list[str]: + return ["_glob_expr"] + + @staticmethod + def _expand_braces(pattern: str) -> list[str]: + """ + Expands brace patterns in a glob string. + For example, "**/*.{js,jsx,ts,tsx}" becomes ["**/*.js", "**/*.jsx", "**/*.ts", "**/*.tsx"]. + Handles multiple brace sets as well. + """ + patterns = [pattern] + while any("{" in p or "}" in p for p in patterns): + new_patterns = [] + for p in patterns: + match = re.search(r"\{([^{}]*)\}", p) + if match: + options_expr = match.group(1) + if not options_expr: + raise ValueError(f"Invalid glob brace expression in {pattern!r}: empty braces are not allowed") + + prefix = p[: match.start()] + suffix = p[match.end() :] + options = options_expr.split(",") + for option in options: + new_patterns.append(f"{prefix}{option}{suffix}") + else: + raise ValueError(f"Invalid glob brace expression in {pattern!r}: unmatched brace") + patterns = new_patterns + return patterns + + @staticmethod + def _translate_glob_to_regex(pattern: str) -> str: + res = [] + i = 0 + n = len(pattern) + while i < n: + c = pattern[i] + i += 1 + if c == "*": + if i < n and pattern[i] == "*": # ** + i += 1 + if i < n and pattern[i] == "/": # **/ + i += 1 + if res and res[-1] == "/": # /**/ + res = res[:-1] + res.append("(?:/|/.*/)") + elif res: # c**/ (with c != '/') + res.append(".*/") + else: # **/ at the start of the pattern + res.append("(?:.*/)?") + else: # c**d (with c, d != '/') + res.append(".*") + else: # single * + if not res or res[-1] != "[^/]*": + res.append("[^/]*") + elif c == "?": + res.append("[^/]") + elif c == "[": + j = i + if j < n and pattern[j] == "!": + j += 1 + if j < n and pattern[j] == "]": + j += 1 + while j < n and pattern[j] != "]": + j += 1 + if j >= n: + res.append(r"\[") + else: + stuff = pattern[i:j] + if stuff.startswith("!"): + stuff = "^" + stuff[1:] + elif stuff.startswith("^"): + stuff = r"\^" + stuff + res.append(f"[{stuff}]") + i = j + 1 + else: + res.append(re.escape(c)) + inner = "".join(res) + return f"(?s:{inner})\\Z" def matches(self, path: str) -> bool: - return any(glob_match(p, path) for p in self._patterns) + path = path.replace("\\", "/") # normalise backslashes to forward slashes + return any(regex.match(path) for regex in self._regex_patterns) def search_files( diff --git a/test/serena/test_text_utils.py b/test/serena/test_text_utils.py index 0eb86f3c..865b71b4 100644 --- a/test/serena/test_text_utils.py +++ b/test/serena/test_text_utils.py @@ -1,10 +1,9 @@ -import re from collections.abc import Callable import pytest from serena.util.file_proxy import FileCollection, FileProxy -from serena.util.text_utils import LineType, MultiFileContentReplacer, glob_to_regex, search_files, search_text +from serena.util.text_utils import GlobMatcher, LineType, MultiFileContentReplacer, search_files, search_text class TestSearchText: @@ -177,94 +176,6 @@ class TestSearchText: match_lines = [line for line in multiline_match.lines if line.match_type == LineType.MATCH] assert len(match_lines) >= 3 - def test_search_text_with_glob_pattern(self): - """Test searching with glob-like patterns.""" - content = """ - class UserService: - def get_user(self, user_id): - return {"id": user_id, "name": "Test User"} - - def create_user(self, user_data): - print(f"Creating user: {user_data}") - return {"id": 123, **user_data} - - def update_user(self, user_id, user_data): - print(f"Updating user {user_id} with {user_data}") - return True - """ - - # Search with a glob pattern for all user methods - matches = search_text("*_user*", content=content, is_glob=True, multiline=False) - - assert len(matches) == 3 - assert "get_user" in matches[0].lines[0].line_content - assert "create_user" in matches[1].lines[0].line_content - assert "update_user" in matches[2].lines[0].line_content - - def test_search_text_with_complex_glob_pattern(self): - """Test searching with more complex glob patterns.""" - content = """ - def process_data(data): - return [transform(item) for item in data] - - def transform(item): - if isinstance(item, dict): - return {k: v.upper() if isinstance(v, str) else v for k, v in item.items()} - elif isinstance(item, list): - return [x * 2 for x in item if isinstance(x, (int, float))] - elif isinstance(item, str): - return item.upper() - else: - return item - """ - - # Search with a simplified glob pattern to find all isinstance occurrences - matches = search_text("*isinstance*", content=content, is_glob=True, multiline=False) - - # Should match lines with isinstance(item, dict) and isinstance(item, list) - assert len(matches) >= 2 - instance_matches = [ - line.line_content - for match in matches - for line in match.lines - if line.match_type == LineType.MATCH and "isinstance(item," in line.line_content - ] - assert len(instance_matches) >= 2 - assert any("isinstance(item, dict)" in line for line in instance_matches) - assert any("isinstance(item, list)" in line for line in instance_matches) - - def test_search_text_glob_with_special_chars(self): - """Glob patterns containing regex special characters should match literally.""" - content = """ - def func_square(): - print("value[42]") - - def func_curly(): - print("value{bar}") - """ - - matches_square = search_text(r"*\[42\]*", content=content, is_glob=True, multiline=False) - assert len(matches_square) == 1 - assert "[42]" in matches_square[0].lines[0].line_content - - matches_curly = search_text("*{bar}*", content=content, is_glob=True, multiline=False) - assert len(matches_curly) == 1 - assert "{bar}" in matches_curly[0].lines[0].line_content - - def test_glob_to_regex_question_matches_single_char(self): - """A glob '?' matches exactly one character, matching glob_match's documented semantics.""" - rx = glob_to_regex("a?c") - assert re.fullmatch(rx, "abc") is not None - assert re.fullmatch(rx, "ac") is None - assert re.fullmatch(rx, "abbc") is None - - def test_search_text_glob_question_single_char(self): - """search_text('c?t', is_glob=True) must match 'cat' (one char), not 'coat' (two chars).""" - content = "value_cat_end\nvalue_coat_end" - matches = search_text("c?t", content=content, is_glob=True, multiline=False) - assert len(matches) == 1 - assert "value_cat_end" in matches[0].lines[0].line_content - def test_search_text_no_matches(self): """Test searching with a pattern that doesn't match anything.""" content = """ @@ -317,7 +228,8 @@ class TestSearchFiles: (["a.py", "b.txt", "c.py"], "match", "*.py", "c.*", ["a.py"], "Include .py, exclude c.*"), # Directory matching - Using pathspec patterns (["main.c", "test/main.c"], "match", "test/*", None, ["test/main.c"], "Include files in test/ subdir"), - (["data/a.csv", "data/b.log"], "match", "data/*", "*.log", ["data/a.csv"], "Include data/*, exclude *.log"), + # A bare `*.log` no longer crosses `/` (see #1732); use `data/*.log` to exclude within data/. + (["data/a.csv", "data/b.log"], "match", "data/*", "data/*.log", ["data/a.csv"], "Include data/*, exclude data/*.log"), (["src/a.py", "tests/b.py"], "match", "src/**", "tests/**", ["src/a.py"], "Include src/**, exclude tests/**"), (["src/mod/a.py", "tests/b.py"], "match", "**/*.py", "tests/**", ["src/mod/a.py"], "Include **/*.py, exclude tests/**"), (["file.py", "dir/file.py"], "match", "dir/*.py", None, ["dir/file.py"], "Include files directly in dir"), @@ -593,23 +505,33 @@ class TestGlobMatch: [ # Basic wildcard patterns ("*.py", "file.py", True), + ("*.py", "src/file.py", False), ("*.py", "file.txt", False), ("*agent.py", "agent.py", True), ("*agent.py", "process_isolated_agent.py", True), + ("*agent.py", "src/agent.py", False), ("*agent.py", "agent_test.py", False), + ("a?b.py", "acb.py", True), + ("a?b.py", "a/b.py", False), + ("src/*.py", "src/file.py", True), + ("src/*.py", "src/sub/file.py", False), # Double asterisk patterns ("**agent.py", "agent.py", True), + ("**/agent.py", "agent.py", True), + ("**/agent.py", "src/serena/agent.py", True), + ("src/**/agent.py", "src/agent.py", True), + ("src/**/agent.py", "src/serena/foo/agent.py", True), + ("src/s**a/agent.py", "src/serena/agent.py", True), + ("src/s**a/agent.py", "src/serena/a/agent.py", True), ("**agent.py", "src/agent.py", True), ("**agent.py", "src/serena/agent.py", True), ("**agent.py", "src/serena/process_isolated_agent.py", True), ("**agent.py", "agent_test.py", False), - # Prefix with double asterisk ("src/**agent.py", "src/agent.py", True), ("src/**agent.py", "src/serena/agent.py", True), ("src/**agent.py", "src/serena/process_isolated_agent.py", True), ("src/**agent.py", "other/agent.py", False), ("src/**agent.py", "src/agent_test.py", False), - # Directory patterns ("src/**", "src/file.py", True), ("src/**", "src/dir/file.py", True), ("src/**", "other/file.py", False), @@ -620,13 +542,20 @@ class TestGlobMatch: # Simple patterns without asterisks ("src/file.py", "src/file.py", True), ("src/file.py", "src/other.py", False), + # Patterns with backslash + ("src\\file.py", "src/file.py", True), + ("src\\file.py", "src\\file.py", True), + ("src\\file.py", "src/other.py", False), + # Patterns with [] + ("file[0-9].py", "file1.py", True), + ("file[0-9].py", "filea.py", False), + ("file[!0-9].py", "filea.py", True), + ("file[!0-9].py", "file1.py", False), ], ) def test_glob_match(self, pattern, path, expected): """Test glob_match function with various patterns.""" - from serena.util.text_utils import glob_match - - assert glob_match(pattern, path) == expected + assert GlobMatcher(pattern).matches(path) == expected class TestExpandBraces: @@ -651,9 +580,7 @@ class TestExpandBraces: ) def test_expand_braces(self, pattern, expected): """Test brace expansion for glob patterns.""" - from serena.util.text_utils import expand_braces - - assert sorted(expand_braces(pattern)) == sorted(expected) + assert sorted(GlobMatcher._expand_braces(pattern)) == sorted(expected) @pytest.mark.parametrize( "pattern", @@ -665,10 +592,8 @@ class TestExpandBraces: ) def test_expand_braces_rejects_malformed_braces(self, pattern): """Malformed brace globs should fail instead of looping forever.""" - from serena.util.text_utils import expand_braces - with pytest.raises(ValueError, match="Invalid glob brace expression"): - expand_braces(pattern) + GlobMatcher._expand_braces(pattern) class TestMultiFileContentReplacer: