From 6e518b5f7cb80d06ec3bdf899b4dc825a770929c Mon Sep 17 00:00:00 2001
From: Dominik Jain <dominik.jain@oraios-ai.de>
Date: Wed, 1 Jul 2026 13:50:53 +0200
Subject: [PATCH 1/5] Refactor search in files/iterative file processing

* Introduce abstractions: FileCollection, FileProxy
* Unify handling of pattern search in project files
  (source code vs. all files)

This prepares for processing of external files,
which can implement FileProxy differently.
---
 src/serena/project.py                         | 28 +++++--
 src/serena/tools/file_tools.py                | 45 +++-------
 src/serena/util/file_proxy.py                 | 83 +++++++++++++++++++
 src/serena/util/text_utils.py                 | 69 ++++++---------
 test/serena/test_text_utils.py                | 37 ++++++---
 test/solidlsp/clojure/test_clojure_basic.py   |  4 +-
 .../elixir/test_elixir_integration.py         |  8 +-
 test/solidlsp/python/test_python_basic.py     | 12 +--
 test/solidlsp/swift/test_swift_basic.py       |  6 +-
 9 files changed, 179 insertions(+), 113 deletions(-)
 create mode 100644 src/serena/util/file_proxy.py

diff --git a/src/serena/project.py b/src/serena/project.py
index 1ddc53ac..e40c2dab 100644
--- a/src/serena/project.py
+++ b/src/serena/project.py
@@ -14,7 +14,8 @@ from serena.config.serena_config import (
 )
 from serena.ls_manager import LanguageServerFactory, LanguageServerManager
 from serena.memories.memory_manager import MemoryManager
-from serena.util.file_system import GitignoreParser, match_path
+from serena.util.file_proxy import FileCollection
+from serena.util.file_system import GitignoreParser, match_path, scan_directory
 from serena.util.text_utils import MatchedConsecutiveLines, search_files
 from solidlsp import SolidLanguageServer
 from solidlsp.ls_config import Language
@@ -344,7 +345,7 @@ class Project(ToStringMixin):
                         )
             return rel_file_paths
 
-    def search_source_files_for_pattern(
+    def search_project_files_for_pattern(
         self,
         pattern: str,
         relative_path: str = "",
@@ -353,6 +354,7 @@ class Project(ToStringMixin):
         paths_include_glob: str | None = None,
         paths_exclude_glob: str | None = None,
         multiline: bool = True,
+        code_files_only: bool = True,
     ) -> list[MatchedConsecutiveLines]:
         """
         Search for a pattern across all (non-ignored) source files
@@ -366,12 +368,26 @@ class Project(ToStringMixin):
         :param multiline: Whether to compile the regex with the DOTALL flag (``.`` matches newlines).
         :return: List of matched consecutive lines with context
         """
-        relative_file_paths = self.gather_source_files(relative_path=relative_path)
+        if code_files_only:
+            relative_file_paths = self.gather_source_files(relative_path=relative_path)
+            file_collection = FileCollection.from_local_project_paths(relative_file_paths, self)
+        else:
+            abs_path = os.path.join(self.project_root, relative_path)
+            if os.path.isfile(abs_path):
+                rel_paths_to_search = [relative_path]
+            else:
+                _dirs, rel_paths_to_search = scan_directory(
+                    path=abs_path,
+                    recursive=True,
+                    is_ignored_dir=self.is_ignored_path,
+                    is_ignored_file=self.is_ignored_path,
+                    relative_to=self.project_root,
+                )
+            file_collection = FileCollection.from_local_project_paths(rel_paths_to_search, self)
+
         return search_files(
-            relative_file_paths,
+            file_collection,
             pattern,
-            root_path=self.project_root,
-            file_reader=self.read_file,
             context_lines_before=context_lines_before,
             context_lines_after=context_lines_after,
             paths_include_glob=paths_include_glob,
diff --git a/src/serena/tools/file_tools.py b/src/serena/tools/file_tools.py
index dd61903a..8a3a3e5e 100644
--- a/src/serena/tools/file_tools.py
+++ b/src/serena/tools/file_tools.py
@@ -20,7 +20,6 @@ from serena.util.text_utils import (
     ReplacementOccurrence,
     expand_braces,
     glob_match,
-    search_files,
 )
 
 
@@ -599,40 +598,16 @@ class SearchForPatternTool(Tool):
         if not os.path.exists(abs_path):
             raise FileNotFoundError(f"Relative path {relative_path} does not exist.")
 
-        if restrict_search_to_code_files:
-            matches = self.project.search_source_files_for_pattern(
-                pattern=substring_pattern,
-                relative_path=relative_path,
-                context_lines_before=context_lines_before,
-                context_lines_after=context_lines_after,
-                paths_include_glob=paths_include_glob.strip(),
-                paths_exclude_glob=paths_exclude_glob.strip(),
-                multiline=multiline,
-            )
-        else:
-            if os.path.isfile(abs_path):
-                rel_paths_to_search = [relative_path]
-            else:
-                _dirs, rel_paths_to_search = scan_directory(
-                    path=abs_path,
-                    recursive=True,
-                    is_ignored_dir=self.project.is_ignored_path,
-                    is_ignored_file=self.project.is_ignored_path,
-                    relative_to=self.get_project_root(),
-                )
-            # TODO (maybe): not super efficient to walk through the files again and filter if glob patterns are provided
-            #   but it probably never matters and this version required no further refactoring
-            matches = search_files(
-                rel_paths_to_search,
-                substring_pattern,
-                context_lines_before=context_lines_before,
-                context_lines_after=context_lines_after,
-                file_reader=self.project.read_file,
-                root_path=self.get_project_root(),
-                paths_include_glob=paths_include_glob,
-                paths_exclude_glob=paths_exclude_glob,
-                multiline=multiline,
-            )
+        matches = self.project.search_project_files_for_pattern(
+            pattern=substring_pattern,
+            relative_path=relative_path,
+            context_lines_before=context_lines_before,
+            context_lines_after=context_lines_after,
+            paths_include_glob=paths_include_glob.strip(),
+            paths_exclude_glob=paths_exclude_glob.strip(),
+            multiline=multiline,
+            code_files_only=restrict_search_to_code_files,
+        )
 
         # group matches by file
         file_to_matches: dict[str, list[str]] = defaultdict(list)
diff --git a/src/serena/util/file_proxy.py b/src/serena/util/file_proxy.py
new file mode 100644
index 00000000..13fc070a
--- /dev/null
+++ b/src/serena/util/file_proxy.py
@@ -0,0 +1,83 @@
+import logging
+import os
+from abc import ABC, abstractmethod
+from collections.abc import Iterator
+from typing import TYPE_CHECKING, Self
+
+if TYPE_CHECKING:
+    from serena.project import Project
+
+log = logging.getLogger(__name__)
+
+
+class FileProxy(ABC):
+    @abstractmethod
+    def get_contents(self) -> str:
+        """:return: the contents of the file as a string."""
+
+    @abstractmethod
+    def get_relative_path(self) -> str:
+        """:return: the relative path reported by Serena (actual relative path or encoded external path)"""
+
+
+class LocalProjectFileProxy(FileProxy):
+    def __init__(self, relative_path: str, project: "Project"):
+        self._relative_path = relative_path
+        self._project = project
+
+    def get_contents(self) -> str:
+        abs_path = os.path.join(self._project.project_root, self._relative_path)
+        with open(abs_path, encoding=self._project.project_config.encoding) as f:
+            return f.read()
+
+    def get_relative_path(self) -> str:
+        return self._relative_path
+
+
+class FileCollection:
+    def __init__(self, file_proxies: list[FileProxy]):
+        self._file_proxies = file_proxies
+
+    def __len__(self) -> int:
+        return len(self._file_proxies)
+
+    def __iter__(self) -> Iterator[FileProxy]:
+        return iter(self._file_proxies)
+
+    @classmethod
+    def from_local_project_paths(cls, relative_paths: list[str], project: "Project") -> Self:
+        return cls([LocalProjectFileProxy(path, project) for path in relative_paths])
+
+    def filter_glob(self, paths_include_glob: str | None = None, paths_exclude_glob: str | None = None) -> "FileCollection":
+        """
+        Filters the collection based on the given patterns.
+        Note: Filtering is applied only to local project files. Other files are always retained.
+
+        :param paths_include_glob: optional glob pattern to include files from the list
+        :param paths_exclude_glob: optional glob pattern to exclude files from the list
+        :return: the filtered collection
+        """
+        from serena.util.text_utils import GlobMatcher
+
+        if paths_include_glob is None and paths_exclude_glob is None:
+            return self
+
+        include_glob_matcher = GlobMatcher(paths_include_glob) if paths_include_glob else None
+        exclude_glob_matcher = GlobMatcher(paths_exclude_glob) if paths_exclude_glob else None
+
+        filtered_files = []
+        for f in self._file_proxies:
+            if isinstance(f, LocalProjectFileProxy):
+                local_file: LocalProjectFileProxy = f
+                path = local_file.get_relative_path()
+                if include_glob_matcher:
+                    if not include_glob_matcher.matches(path):
+                        log.debug(f"Skipping {path}: does not match include pattern {paths_include_glob}")
+                        continue
+                if exclude_glob_matcher:
+                    if exclude_glob_matcher.matches(path):
+                        log.debug(f"Skipping {path}: matches exclude pattern {paths_exclude_glob}")
+                        continue
+            filtered_files.append(f)
+
+        return FileCollection(filtered_files)
diff --git a/src/serena/util/text_utils.py b/src/serena/util/text_utils.py
index aff92fdb..b8020e12 100644
--- a/src/serena/util/text_utils.py
+++ b/src/serena/util/text_utils.py
@@ -1,7 +1,6 @@
 import fnmatch
 import hashlib
 import logging
-import os
 import re
 from collections.abc import Callable
 from dataclasses import dataclass, field
@@ -11,7 +10,7 @@ from typing import Any, Literal, Self
 from bs4 import BeautifulSoup
 from joblib import Parallel, delayed
 
-from serena.constants import DEFAULT_SOURCE_FILE_ENCODING
+from serena.util.file_proxy import FileCollection, FileProxy
 from solidlsp.ls_utils import TextUtils
 
 log = logging.getLogger(__name__)
@@ -237,12 +236,6 @@ def search_text(
     return matches
 
 
-def default_file_reader(file_path: str) -> str:
-    """Reads using the default encoding."""
-    with open(file_path, encoding=DEFAULT_SOURCE_FILE_ENCODING) as f:
-        return f.read()
-
-
 def expand_braces(pattern: str) -> list[str]:
     """
     Expands brace patterns in a glob string.
@@ -319,11 +312,17 @@ def glob_match(pattern: str, path: str) -> bool:
         return fnmatch.fnmatch(path, pattern)
 
 
+class GlobMatcher:
+    def __init__(self, expr: str):
+        self._patterns = expand_braces(expr)
+
+    def matches(self, path: str) -> bool:
+        return any(glob_match(p, path) for p in self._patterns)
+
+
 def search_files(
-    relative_file_paths: list[str],
+    file_collection: FileCollection,
     pattern: str,
-    root_path: str = "",
-    file_reader: Callable[[str], str] = default_file_reader,
     context_lines_before: int = 0,
     context_lines_after: int = 0,
     paths_include_glob: str | None = None,
@@ -333,11 +332,8 @@ def search_files(
     """
     Search for a pattern in a list of files.
 
-    :param relative_file_paths: list of relative file paths in which to search
+    :param file_collection: the collection of files to search (will be optionally filtered by glob patterns)
     :param pattern: pattern to search for
-    :param root_path: root path to resolve relative paths against (by default, current working directory).
-    :param file_reader: function to read a file, by default will just use os.open.
-        All files that can't be read by it will be skipped.
     :param context_lines_before: number of context lines to include before matches
     :param context_lines_after: number of context lines to include after matches
     :param paths_include_glob: optional glob pattern to include files from the list
@@ -345,53 +341,36 @@ def search_files(
     :param multiline: whether to apply multi-line matching, enabling the flags re.DOTALL and re.MULTILINE (default: True)
     :return: list of MatchedConsecutiveLines objects
     """
-    # Pre-filter paths (done sequentially to avoid overhead)
-    # Use proper glob matching instead of gitignore patterns
-    include_patterns = expand_braces(paths_include_glob) if paths_include_glob else None
-    exclude_patterns = expand_braces(paths_exclude_glob) if paths_exclude_glob else None
+    # apply glob filter
+    file_collection = file_collection.filter_glob(paths_include_glob=paths_include_glob, paths_exclude_glob=paths_exclude_glob)
+    log.info(f"Processing {len(file_collection)} files.")
 
-    filtered_paths = []
-    for path in relative_file_paths:
-        if include_patterns:
-            if not any(glob_match(p, path) for p in include_patterns):
-                log.debug(f"Skipping {path}: does not match include pattern {paths_include_glob}")
-                continue
-
-        if exclude_patterns:
-            if any(glob_match(p, path) for p in exclude_patterns):
-                log.debug(f"Skipping {path}: matches exclude pattern {paths_exclude_glob}")
-                continue
-
-        filtered_paths.append(path)
-
-    log.info(f"Processing {len(filtered_paths)} files.")
-
-    def process_single_file(path: str) -> dict[str, Any]:
+    def process_single_file(file_proxy: FileProxy) -> dict[str, Any]:
         """Process a single file - this function will be parallelized."""
+        relative_path = file_proxy.get_relative_path()
         try:
-            abs_path = os.path.join(root_path, path)
-            file_content = file_reader(abs_path)
+            file_content = file_proxy.get_contents()
             search_results = search_text(
                 pattern,
                 content=file_content,
-                source_file_path=path,
+                source_file_path=relative_path,
                 allow_multiline_match=True,
                 context_lines_before=context_lines_before,
                 context_lines_after=context_lines_after,
                 multiline=multiline,
             )
             if len(search_results) > 0:
-                log.debug(f"Found {len(search_results)} matches in {path}")
-            return {"path": path, "results": search_results, "error": None}
+                log.debug(f"Found {len(search_results)} matches in {relative_path}")
+            return {"path": relative_path, "results": search_results, "error": None}
         except Exception as e:
-            log.debug(f"Error processing {path}: {e}")
-            return {"path": path, "results": [], "error": str(e)}
+            log.debug(f"Error processing {relative_path}: {e}")
+            return {"path": relative_path, "results": [], "error": str(e)}
 
     # Execute in parallel using joblib
     results = Parallel(
         n_jobs=-1,
         backend="threading",
-    )(delayed(process_single_file)(path) for path in filtered_paths)
+    )(delayed(process_single_file)(file_proxy) for file_proxy in file_collection)
 
     # Collect results and errors
     matches = []
@@ -406,7 +385,7 @@ def search_files(
     if skipped_file_error_tuples:
         log.debug(f"Failed to read {len(skipped_file_error_tuples)} files: {skipped_file_error_tuples}")
 
-    log.info(f"Found {len(matches)} total matches across {len(filtered_paths)} files")
+    log.info(f"Found {len(matches)} total matches across {len(file_collection)} files")
     return matches
 
 
diff --git a/test/serena/test_text_utils.py b/test/serena/test_text_utils.py
index e6cd4539..408aadac 100644
--- a/test/serena/test_text_utils.py
+++ b/test/serena/test_text_utils.py
@@ -1,7 +1,9 @@
 import re
+from collections.abc import Callable
 
 import pytest
 
+from serena.util.file_proxy import FileCollection, FileProxy
 from serena.util.text_utils import LineType, MultiFileContentReplacer, search_files, search_text
 
 
@@ -221,6 +223,23 @@ def mock_reader_always_match(file_path: str) -> str:
     return "This line contains a match."
 
 
+class MockFileProxy(FileProxy):
+    def __init__(self, relative_path: str, mock_reader: Callable[[str], str] = mock_reader_always_match):
+        self.relative_path = relative_path
+        self.mock_reader = mock_reader
+
+    def get_contents(self) -> str:
+        return self.mock_reader(self.relative_path)
+
+    def get_relative_path(self) -> str:
+        return self.relative_path
+
+
+class MockFileCollection(FileCollection):
+    def __init__(self, file_paths, mock_reader: Callable[[str], str] = mock_reader_always_match):
+        super().__init__([MockFileProxy(path, mock_reader) for path in file_paths])
+
+
 class TestSearchFiles:
     @pytest.mark.parametrize(
         "file_paths, pattern, paths_include_glob, paths_exclude_glob, expected_matched_files, description",
@@ -254,9 +273,8 @@ class TestSearchFiles:
         Test the include/exclude glob filtering logic in search_files using PathSpec patterns.
         """
         results = search_files(
-            relative_file_paths=file_paths,
+            MockFileCollection(file_paths),
             pattern=pattern,
-            file_reader=mock_reader_always_match,
             paths_include_glob=paths_include_glob,
             paths_exclude_glob=paths_exclude_glob,
             context_lines_before=0,  # No context needed for this test focus
@@ -333,9 +351,8 @@ class TestSearchFiles:
         Test glob patterns that were problematic with the previous gitignore-based implementation.
         """
         results = search_files(
-            relative_file_paths=file_paths,
+            MockFileCollection(file_paths),
             pattern=pattern,
-            file_reader=mock_reader_always_match,
             paths_include_glob=paths_include_glob,
             paths_exclude_glob=paths_exclude_glob,
             context_lines_before=0,
@@ -414,9 +431,8 @@ class TestSearchFiles:
     ):
         """Test search_files with glob patterns containing brace expansions."""
         results = search_files(
-            relative_file_paths=file_paths,
+            MockFileCollection(file_paths),
             pattern=pattern,
-            file_reader=mock_reader_always_match,
             paths_include_glob=paths_include_glob,
             paths_exclude_glob=paths_exclude_glob,
         )
@@ -429,9 +445,8 @@ class TestSearchFiles:
         file_paths = ["a.py", "b.txt"]
         pattern = "non_existent_pattern_in_mock_content"  # This won't match mock_reader_always_match content
         results = search_files(
-            relative_file_paths=file_paths,
+            MockFileCollection(file_paths),
             pattern=pattern,
-            file_reader=mock_reader_always_match,  # Content is "This line contains a match."
             paths_include_glob=None,  # Both files would pass filters
             paths_exclude_glob=None,
         )
@@ -454,9 +469,8 @@ class TestSearchFiles:
         pattern = r"value=(\d+)"
 
         results = search_files(
-            relative_file_paths=file_paths,
+            MockFileCollection(file_paths, specific_mock_reader),
             pattern=pattern,
-            file_reader=specific_mock_reader,
             paths_include_glob="*.py",  # Only include .py files
             paths_exclude_glob="b.*",  # Exclude files starting with b
         )
@@ -484,9 +498,8 @@ class TestSearchFiles:
         pattern = "MATCH HERE"
 
         results = search_files(
-            relative_file_paths=file_paths,
+            MockFileCollection(file_paths, context_mock_reader),
             pattern=pattern,
-            file_reader=context_mock_reader,
             paths_include_glob="*.txt",  # Only include .txt files
             paths_exclude_glob=None,
             context_lines_before=1,
diff --git a/test/solidlsp/clojure/test_clojure_basic.py b/test/solidlsp/clojure/test_clojure_basic.py
index c42067ac..3dd06d8e 100644
--- a/test/solidlsp/clojure/test_clojure_basic.py
+++ b/test/solidlsp/clojure/test_clojure_basic.py
@@ -211,7 +211,7 @@ class TestProjectBasics:
 
     @pytest.mark.parametrize("project", [Language.CLOJURE], indirect=True)
     def test_search_files_for_pattern(self, project: Project) -> None:
-        result = project.search_source_files_for_pattern("defn.*greet")
+        result = project.search_project_files_for_pattern("defn.*greet")
 
         assert result is not None, "Pattern search should return results"
         assert len(result) > 0, "Should find at least one match for 'defn.*greet'"
@@ -219,7 +219,7 @@ class TestProjectBasics:
         core_matches = [match for match in result if match.source_file_path and "core.clj" in match.source_file_path]
         assert len(core_matches) > 0, "Should find greet function in core.clj"
 
-        result = project.search_source_files_for_pattern(":require")
+        result = project.search_project_files_for_pattern(":require")
 
         assert result is not None, "Should find require statements"
         utils_matches = [match for match in result if match.source_file_path and "utils.clj" in match.source_file_path]
diff --git a/test/solidlsp/elixir/test_elixir_integration.py b/test/solidlsp/elixir/test_elixir_integration.py
index 678ce762..1658993f 100644
--- a/test/solidlsp/elixir/test_elixir_integration.py
+++ b/test/solidlsp/elixir/test_elixir_integration.py
@@ -115,7 +115,7 @@ class TestElixirProject:
         """Test comprehensive symbol search across the entire project."""
         # Search for all function definitions
         function_pattern = r"def\s+\w+\s*[\(\s]"
-        function_matches = project.search_source_files_for_pattern(function_pattern)
+        function_matches = project.search_project_files_for_pattern(function_pattern)
 
         # Should find functions across multiple files
         if function_matches:
@@ -131,7 +131,7 @@ class TestElixirProject:
 
         # Search for struct definitions
         struct_pattern = r"defstruct\s+\["
-        struct_matches = project.search_source_files_for_pattern(struct_pattern)
+        struct_matches = project.search_project_files_for_pattern(struct_pattern)
 
         if struct_matches:
             # Should find structs primarily in models.ex
@@ -143,7 +143,7 @@ class TestElixirProject:
         """Test that the language server understands Elixir protocols and implementations."""
         # Search for protocol definitions
         protocol_pattern = r"defprotocol\s+\w+"
-        protocol_matches = project.search_source_files_for_pattern(protocol_pattern, paths_include_glob="**/models.ex")
+        protocol_matches = project.search_project_files_for_pattern(protocol_pattern, paths_include_glob="**/models.ex")
 
         if protocol_matches:
             # Should find the Serializable protocol
@@ -152,7 +152,7 @@ class TestElixirProject:
 
         # Search for protocol implementations
         impl_pattern = r"defimpl\s+\w+"
-        impl_matches = project.search_source_files_for_pattern(impl_pattern, paths_include_glob="**/models.ex")
+        impl_matches = project.search_project_files_for_pattern(impl_pattern, paths_include_glob="**/models.ex")
 
         if impl_matches:
             # Should find multiple implementations
diff --git a/test/solidlsp/python/test_python_basic.py b/test/solidlsp/python/test_python_basic.py
index f3d46f0f..283e3e56 100644
--- a/test/solidlsp/python/test_python_basic.py
+++ b/test/solidlsp/python/test_python_basic.py
@@ -214,28 +214,28 @@ class TestProjectBasics:
         """Test search_files_for_pattern with various patterns and glob filters."""
         # Test 1: Search for class definitions across all files
         class_pattern = r"class\s+\w+\s*(?:\([^{]*\)|:)"
-        matches = project.search_source_files_for_pattern(class_pattern)
+        matches = project.search_project_files_for_pattern(class_pattern)
         assert len(matches) > 0
         # Should find multiple classes like User, Item, BaseModel, etc.
         assert len(matches) >= 5
 
         # Test 2: Search for specific class with include glob
         user_class_pattern = r"class\s+User\s*(?:\([^{]*\)|:)"
-        matches = project.search_source_files_for_pattern(user_class_pattern, paths_include_glob="**/models.py")
+        matches = project.search_project_files_for_pattern(user_class_pattern, paths_include_glob="**/models.py")
         assert len(matches) == 1  # Should only find User class in models.py
         assert matches[0].source_file_path is not None
         assert "models.py" in matches[0].source_file_path
 
         # Test 3: Search for method definitions with exclude glob
         method_pattern = r"def\s+\w+\s*\([^)]*\):"
-        matches = project.search_source_files_for_pattern(method_pattern, paths_exclude_glob="**/models.py")
+        matches = project.search_project_files_for_pattern(method_pattern, paths_exclude_glob="**/models.py")
         assert len(matches) > 0
         # Should find methods in services.py but not in models.py
         assert all(match.source_file_path is not None and "models.py" not in match.source_file_path for match in matches)
 
         # Test 4: Search for specific method with both include and exclude globs
         create_user_pattern = r"def\s+create_user\s*\([^)]*\)(?:\s*->[^:]+)?:"
-        matches = project.search_source_files_for_pattern(
+        matches = project.search_project_files_for_pattern(
             create_user_pattern, paths_include_glob="**/*.py", paths_exclude_glob="**/models.py"
         )
         assert len(matches) == 1  # Should only find create_user in services.py
@@ -244,7 +244,7 @@ class TestProjectBasics:
 
         # Test 5: Search for a pattern that should appear in multiple files
         init_pattern = r"def\s+__init__\s*\([^)]*\):"
-        matches = project.search_source_files_for_pattern(init_pattern)
+        matches = project.search_project_files_for_pattern(init_pattern)
         assert len(matches) > 1  # Should find __init__ in multiple classes
         # Should find __init__ in both models.py and services.py
         assert any(match.source_file_path is not None and "models.py" in match.source_file_path for match in matches)
@@ -252,7 +252,7 @@ class TestProjectBasics:
 
         # Test 6: Search with a pattern that should have no matches
         no_match_pattern = r"def\s+this_method_does_not_exist\s*\([^)]*\):"
-        matches = project.search_source_files_for_pattern(no_match_pattern)
+        matches = project.search_project_files_for_pattern(no_match_pattern)
         assert len(matches) == 0
 
     @pytest.mark.parametrize("language_server", PYTHON_BACKEND_LANGUAGES, indirect=True)
diff --git a/test/solidlsp/swift/test_swift_basic.py b/test/solidlsp/swift/test_swift_basic.py
index a2282fb2..f4395f4c 100644
--- a/test/solidlsp/swift/test_swift_basic.py
+++ b/test/solidlsp/swift/test_swift_basic.py
@@ -198,7 +198,7 @@ class TestSwiftProjectBasics:
 
         # Scenario 3: Search for struct definitions
         struct_pattern = r"struct\s+\w+"
-        matches = project.search_source_files_for_pattern(struct_pattern)
+        matches = project.search_project_files_for_pattern(struct_pattern)
         assert len(matches) > 0, "Should find struct definitions"
         # Should find User struct
         user_matches = [m for m in matches if "User" in str(m)]
@@ -206,7 +206,7 @@ class TestSwiftProjectBasics:
 
         # Scenario 4: Search for class definitions
         class_pattern = r"class\s+\w+"
-        matches = project.search_source_files_for_pattern(class_pattern)
+        matches = project.search_project_files_for_pattern(class_pattern)
         assert len(matches) > 0, "Should find class definitions"
         # Should find Calculator and Circle classes
         calculator_matches = [m for m in matches if "Calculator" in str(m)]
@@ -216,7 +216,7 @@ class TestSwiftProjectBasics:
 
         # Scenario 5: Search for enum definitions
         enum_pattern = r"enum\s+\w+"
-        matches = project.search_source_files_for_pattern(enum_pattern)
+        matches = project.search_project_files_for_pattern(enum_pattern)
         assert len(matches) > 0, "Should find enum definitions"
         # Should find Status enum
         status_matches = [m for m in matches if "Status" in str(m)]

From f39906728da524fd0d1669466ad1c0de9685682c Mon Sep 17 00:00:00 2001
From: Dominik Jain <dominik.jain@oraios-ai.de>
Date: Wed, 1 Jul 2026 13:56:41 +0200
Subject: [PATCH 2/5] search_text: Remove obsolete option allow_multiline_match

The obsolete option was only used in tests, where adjusting the `multiline` parameter
restored the tested functionality.
---
 src/serena/util/text_utils.py  | 76 +++++++++++-----------------------
 test/serena/test_text_utils.py | 17 ++++----
 2 files changed, 32 insertions(+), 61 deletions(-)

diff --git a/src/serena/util/text_utils.py b/src/serena/util/text_utils.py
index b8020e12..67110f70 100644
--- a/src/serena/util/text_utils.py
+++ b/src/serena/util/text_utils.py
@@ -141,7 +141,6 @@ def search_text(
     pattern: str,
     content: str | None = None,
     source_file_path: str | None = None,
-    allow_multiline_match: bool = False,
     context_lines_before: int = 0,
     context_lines_after: int = 0,
     is_glob: bool = False,
@@ -154,8 +153,6 @@ def search_text(
     :param content: The text content to search. May be None if source_file_path is provided.
     :param source_file_path: Optional path to the source file. If content is None,
         this has to be passed and the file will be read.
-    :param allow_multiline_match: Whether to search across multiple lines. Currently, the default
-        option (False) is very inefficient, so it is recommended to set this to True.
     :param context_lines_before: Number of context lines to include before matches
     :param context_lines_after: Number of context lines to include after matches
     :param is_glob: If True, pattern is treated as a glob-like pattern (e.g., "*.py", "test_??.py")
@@ -178,60 +175,36 @@ def search_text(
     # Convert pattern to a compiled regex if it's a string
     if is_glob:
         pattern = glob_to_regex(pattern)
-    if allow_multiline_match:
-        # For multiline matches, optionally use DOTALL so '.' matches newlines
-        flags = (re.MULTILINE | re.DOTALL) if multiline else 0
-        compiled_pattern = re.compile(pattern, flags)
-        # Search across the entire content as a single string
-        for match in compiled_pattern.finditer(content):
-            start_pos = match.start()
-            end_pos = match.end()
 
-            # Find the line numbers for the start and end positions
-            start_line_num = content[:start_pos].count("\n")
-            end_line_num = content[:end_pos].count("\n")
+    # For multiline matches, optionally use DOTALL so '.' matches newlines
+    flags = (re.MULTILINE | re.DOTALL) if multiline else 0
+    compiled_pattern = re.compile(pattern, flags)
+    # Search across the entire content as a single string
+    for match in compiled_pattern.finditer(content):
+        start_pos = match.start()
+        end_pos = match.end()
 
-            # Calculate the range of lines to include in the context
-            context_start = max(0, start_line_num - context_lines_before)
-            context_end = min(total_lines - 1, end_line_num + context_lines_after)
+        # Find the line numbers for the start and end positions
+        start_line_num = content[:start_pos].count("\n")
+        end_line_num = content[:end_pos].count("\n")
 
-            # Create TextLine objects for the context
-            context_lines = []
-            for line_num in range(context_start, context_end + 1):
-                if context_start <= line_num < start_line_num:
-                    match_type = LineType.BEFORE_MATCH
-                elif end_line_num < line_num <= context_end:
-                    match_type = LineType.AFTER_MATCH
-                else:
-                    match_type = LineType.MATCH
+        # Calculate the range of lines to include in the context
+        context_start = max(0, start_line_num - context_lines_before)
+        context_end = min(total_lines - 1, end_line_num + context_lines_after)
 
-                context_lines.append(TextLine(line_number=line_num, line_content=lines[line_num], match_type=match_type))
+        # Create TextLine objects for the context
+        context_lines = []
+        for line_num in range(context_start, context_end + 1):
+            if context_start <= line_num < start_line_num:
+                match_type = LineType.BEFORE_MATCH
+            elif end_line_num < line_num <= context_end:
+                match_type = LineType.AFTER_MATCH
+            else:
+                match_type = LineType.MATCH
 
-            matches.append(MatchedConsecutiveLines(lines=context_lines, source_file_path=source_file_path))
-    else:
-        # TODO: extremely inefficient! Since we currently don't use this option in SerenaAgent or LanguageServer,
-        #   it is not urgent to fix, but should be either improved or the option should be removed.
-        # Search line by line, normal compile without DOTALL
-        compiled_pattern = re.compile(pattern)
-        for i, line in enumerate(lines):
-            if compiled_pattern.search(line):
-                # Calculate the range of lines to include in the context
-                context_start = max(0, i - context_lines_before)
-                context_end = min(total_lines - 1, i + context_lines_after)
+            context_lines.append(TextLine(line_number=line_num, line_content=lines[line_num], match_type=match_type))
 
-                # Create TextLine objects for the context
-                context_lines = []
-                for j in range(context_start, context_end + 1):
-                    if j < i:
-                        match_type = LineType.BEFORE_MATCH
-                    elif j > i:
-                        match_type = LineType.AFTER_MATCH
-                    else:
-                        match_type = LineType.MATCH
-
-                    context_lines.append(TextLine(line_number=j, line_content=lines[j], match_type=match_type))
-
-                matches.append(MatchedConsecutiveLines(lines=context_lines, source_file_path=source_file_path))
+        matches.append(MatchedConsecutiveLines(lines=context_lines, source_file_path=source_file_path))
 
     return matches
 
@@ -354,7 +327,6 @@ def search_files(
                 pattern,
                 content=file_content,
                 source_file_path=relative_path,
-                allow_multiline_match=True,
                 context_lines_before=context_lines_before,
                 context_lines_after=context_lines_after,
                 multiline=multiline,
diff --git a/test/serena/test_text_utils.py b/test/serena/test_text_utils.py
index 408aadac..bdfa6e76 100644
--- a/test/serena/test_text_utils.py
+++ b/test/serena/test_text_utils.py
@@ -1,4 +1,3 @@
-import re
 from collections.abc import Callable
 
 import pytest
@@ -49,7 +48,7 @@ class TestSearchText:
         assert "def process" in matches[1].lines[0].line_content
         assert "def filter" in matches[2].lines[0].line_content
 
-    def test_search_text_with_compiled_regex(self):
+    def test_search_text_with_regex_pattern2(self):
         """Test searching with a pre-compiled regex pattern."""
         content = """
         import os
@@ -66,8 +65,8 @@ class TestSearchText:
         """
 
         # Search for variable assignments with a compiled regex
-        pattern = re.compile(r"^\s*[A-Z_]+ = .+$")
-        matches = search_text(pattern, content=content)
+        pattern = r"^\s*[A-Z_]+ = .*?$"
+        matches = search_text(pattern, content=content, multiline=True)
 
         assert len(matches) == 2
         assert "DEBUG = True" in matches[0].lines[0].line_content
@@ -117,7 +116,7 @@ class TestSearchText:
 
         # Search for a pattern that spans multiple lines (if-else block)
         pattern = r"if.*?else.*?return"
-        matches = search_text(pattern, content=content, allow_multiline_match=True)
+        matches = search_text(pattern, content=content)
 
         assert len(matches) == 1
         multiline_match = matches[0]
@@ -145,7 +144,7 @@ class TestSearchText:
         """
 
         # Search with a glob pattern for all user methods
-        matches = search_text("*_user*", content=content, is_glob=True)
+        matches = search_text("*_user*", content=content, is_glob=True, multiline=False)
 
         assert len(matches) == 3
         assert "get_user" in matches[0].lines[0].line_content
@@ -170,7 +169,7 @@ class TestSearchText:
         """
 
         # Search with a simplified glob pattern to find all isinstance occurrences
-        matches = search_text("*isinstance*", content=content, is_glob=True)
+        matches = search_text("*isinstance*", content=content, is_glob=True, multiline=False)
 
         # Should match lines with isinstance(item, dict) and isinstance(item, list)
         assert len(matches) >= 2
@@ -194,11 +193,11 @@ class TestSearchText:
             print("value{bar}")
         """
 
-        matches_square = search_text(r"*\[42\]*", content=content, is_glob=True)
+        matches_square = search_text(r"*\[42\]*", content=content, is_glob=True, multiline=False)
         assert len(matches_square) == 1
         assert "[42]" in matches_square[0].lines[0].line_content
 
-        matches_curly = search_text("*{bar}*", content=content, is_glob=True)
+        matches_curly = search_text("*{bar}*", content=content, is_glob=True, multiline=False)
         assert len(matches_curly) == 1
         assert "{bar}" in matches_curly[0].lines[0].line_content
 

From 423c04101d99c5ec9f74135d44ffdc22c82b5a4e Mon Sep 17 00:00:00 2001
From: Dominik Jain <dominik.jain@oraios-ai.de>
Date: Wed, 1 Jul 2026 14:15:16 +0200
Subject: [PATCH 3/5] Make Project aware of the language backend (as it can
 affect its behaviour)

---
 src/serena/agent.py                | 10 ++++------
 src/serena/config/serena_config.py | 11 +++++++++++
 src/serena/project.py              | 11 +++++++++++
 3 files changed, 26 insertions(+), 6 deletions(-)

diff --git a/src/serena/agent.py b/src/serena/agent.py
index d5b69269..953e4241 100644
--- a/src/serena/agent.py
+++ b/src/serena/agent.py
@@ -640,12 +640,10 @@ class SerenaAgent:
         # determine the effective language backend for this session.
         # If a startup project is provided and has a per-project override, use it; otherwise use the global config.
         # Since we don't want to change the toolset after startup, the language backend cannot be changed within a running Serena session
-        self._language_backend = self.serena_config.language_backend
-        if registered_project_to_activate is not None and registered_project_to_activate.project_config.language_backend is not None:
-            self._language_backend = registered_project_to_activate.project_config.language_backend
-            log.info(f"Using language backend as configured in project.yml: {self._language_backend.name}")
-        else:
-            log.info(f"Using language backend from global configuration: {self._language_backend.name}")
+        self._language_backend = self.serena_config.determine_language_backend(
+            project_config=registered_project_to_activate.project_config if registered_project_to_activate is not None else None,
+            log_choice=True,
+        )
 
         # create the tool names mapping for prompts
         self._prompt_tool_names_mapping = self._create_prompt_tool_names_mapping(self._language_backend)
diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py
index 0b8fc673..2b580082 100644
--- a/src/serena/config/serena_config.py
+++ b/src/serena/config/serena_config.py
@@ -1297,3 +1297,14 @@ class SerenaConfig(SharedConfig, ModeSelectionDefinitionWithBaseModes):
             if glob_match(pattern, project_root_str):
                 return True
         return False
+
+    def determine_language_backend(self, project_config: ProjectConfig | None = None, log_choice: bool = False):
+        language_backend = self.language_backend
+        if project_config and project_config.language_backend is not None:
+            language_backend = project_config.language_backend
+            if log_choice:
+                log.info(f"Using language backend as configured in project: {language_backend.name}")
+        else:
+            if log_choice:
+                log.info(f"Using language backend from global configuration: {language_backend.name}")
+        return language_backend
diff --git a/src/serena/project.py b/src/serena/project.py
index e40c2dab..87d0f969 100644
--- a/src/serena/project.py
+++ b/src/serena/project.py
@@ -9,6 +9,7 @@ from sensai.util.logging import LogTime
 from sensai.util.string import TextBuilder, ToStringMixin
 
 from serena.config.serena_config import (
+    LanguageBackend,
     ProjectConfig,
     SerenaConfig,
 )
@@ -121,6 +122,16 @@ class Project(ToStringMixin):
     def project_name(self) -> str:
         return self.project_config.project_name
 
+    @property
+    def language_backend(self) -> LanguageBackend:
+        # The backend configuration is fundamentally owned by the agent, so it takes
+        # precedence. (Note: The agent does not necessary honour the project's choice,
+        # as it may be invalid.)
+        if self._agent is not None:
+            return self._agent.get_language_backend()
+        else:
+            return self.serena_config.determine_language_backend(self.project_config)
+
     @classmethod
     def load(
         cls,

From 3096ac4fe2567c11f3156bd65c51d4c435de3afd Mon Sep 17 00:00:00 2001
From: Dominik Jain <dominik.jain@oraios-ai.de>
Date: Wed, 1 Jul 2026 14:19:25 +0200
Subject: [PATCH 4/5] Fix in Project._is_ignored_relative_path: Filtering of
 files for source files ignored the language backend

The check is really only possible for LSP,
so the flag is now ignored for JetBrains (where there is no known list of languages).
---
 CHANGELOG.md          |  2 ++
 src/serena/project.py | 25 ++++++++++++++-----------
 2 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c12bb3ec..8a26a59a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -18,6 +18,8 @@ Status of the `main` branch. Changes prior to the next official version change w
   - Update prompts/instructions: Serena instructions manual, modes (editing, interactive) 
   - Allow structured tool output to be configured on a per-context basis, disabling it for Claude Code
     (which does not correctly unpack structured output) #1042
+  - Fix: Project-specific filtering of files for source files ignored the language backend. 
+    The check is really only possible for LSP. 
 
 * CLI:
   - Fix `--project-from-cwd` hijacking git worktrees nested under a Serena project. `find_project_root`
diff --git a/src/serena/project.py b/src/serena/project.py
index 87d0f969..8152a52a 100644
--- a/src/serena/project.py
+++ b/src/serena/project.py
@@ -231,17 +231,20 @@ class Project(ToStringMixin):
             log.debug(f"Path {abs_path} does not exist, skipping ignore check")
             return False
 
-        # Check file extension if it's a file
-        is_file = os.path.isfile(abs_path)
-        if is_file and ignore_non_source_files:
-            is_file_in_supported_language = False
-            for language in self.project_config.languages:
-                fn_matcher = language.get_source_fn_matcher()
-                if fn_matcher.is_relevant_filename(abs_path):
-                    is_file_in_supported_language = True
-                    break
-            if not is_file_in_supported_language:
-                return True
+        # check code file restriction (depending on backend)
+        if ignore_non_source_files:
+            # apply restriction only for LSP backend, which enumerates known languages
+            # and therefore can determine whether a file is a source file or not
+            if self.language_backend.is_lsp():
+                if os.path.isfile(abs_path):
+                    is_file_in_supported_language = False
+                    for language in self.project_config.languages:
+                        fn_matcher = language.get_source_fn_matcher()
+                        if fn_matcher.is_relevant_filename(abs_path):
+                            is_file_in_supported_language = True
+                            break
+                    if not is_file_in_supported_language:
+                        return True
 
         # Create normalized path for consistent handling
         rel_path = Path(relative_path)

From d5d88d72eb3379115fa8f4e3895d5c4dd28d9d3a Mon Sep 17 00:00:00 2001
From: Dominik Jain <dominik.jain@oraios-ai.de>
Date: Wed, 1 Jul 2026 16:55:36 +0200
Subject: [PATCH 5/5] Improve handling of glob support in FileProxy, fixing
 tests

---
 src/serena/util/file_proxy.py  | 14 +++++++++++---
 test/serena/test_text_utils.py |  3 +++
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/serena/util/file_proxy.py b/src/serena/util/file_proxy.py
index 13fc070a..53685dc8 100644
--- a/src/serena/util/file_proxy.py
+++ b/src/serena/util/file_proxy.py
@@ -19,6 +19,12 @@ class FileProxy(ABC):
     def get_relative_path(self) -> str:
         """:return: the relative path reported by Serena (actual relative path or encoded external path)"""
 
+    @abstractmethod
+    def is_glob_supported(self):
+        """
+        :return: whether the proxy supports glob filtering based on its relative path
+        """
+
 
 class LocalProjectFileProxy(FileProxy):
     def __init__(self, relative_path: str, project: "Project"):
@@ -33,6 +39,9 @@ class LocalProjectFileProxy(FileProxy):
     def get_relative_path(self) -> str:
         return self._relative_path
 
+    def is_glob_supported(self):
+        return True
+
 
 class FileCollection:
     def __init__(self, file_proxies: list[FileProxy]):
@@ -67,9 +76,8 @@ class FileCollection:
 
         filtered_files = []
         for f in self._file_proxies:
-            if isinstance(f, LocalProjectFileProxy):
-                local_file: LocalProjectFileProxy = f
-                path = local_file.get_relative_path()
+            if f.is_glob_supported():
+                path = f.get_relative_path()
                 if include_glob_matcher:
                     if not include_glob_matcher.matches(path):
                         log.debug(f"Skipping {path}: does not match include pattern {paths_include_glob}")
diff --git a/test/serena/test_text_utils.py b/test/serena/test_text_utils.py
index bdfa6e76..71b0aad6 100644
--- a/test/serena/test_text_utils.py
+++ b/test/serena/test_text_utils.py
@@ -233,6 +233,9 @@ class MockFileProxy(FileProxy):
     def get_relative_path(self) -> str:
         return self.relative_path
 
+    def is_glob_supported(self):
+        return True
+
 
 class MockFileCollection(FileCollection):
     def __init__(self, file_paths, mock_reader: Callable[[str], str] = mock_reader_always_match):