Extended pattern search tool to also work on non-code files

README.md

@@ -597,7 +597,7 @@ Here is the full list of Serena's tools with a short description (output of `uv
 * `replace_lines`: Replaces a range of lines within a file with new content.
 * `replace_symbol_body`: Replaces the full definition of a symbol.
 * `restart_language_server`: Restarts the language server, may be necessary when edits not through Serena happen.
-* `search_in_all_code`: Performs a search for a pattern in all code files (and only in code files) in the project.
+* `search_for_pattern`: Performs a search for a pattern in the project.
 * `summarize_changes`: Provides instructions for summarizing the changes made to the codebase.
 * `think_about_collected_information`: Thinking tool for pondering the completeness of collected information.
 * `think_about_task_adherence`: Thinking tool for determining whether the agent is still on track with the current task.

src/multilspy/language_server.py

@@ -244,10 +244,22 @@ class LanguageServer:
                     processed_patterns.append(line.strip())
 
         # Create a pathspec matcher from the processed patterns
-        self.ignore_spec = pathspec.PathSpec.from_lines(
+        self._ignore_spec = pathspec.PathSpec.from_lines(
             pathspec.patterns.GitWildMatchPattern,
             processed_patterns
         )
+        
+    def get_ignore_spec(self) -> pathspec.PathSpec:
+        """Returns the pathspec matcher for the paths that were configured to be ignored through
+        the multilspy config file and the .gitignore file.
+        
+        This is is a subset of the full language-specific ignore spec that determines
+        which files are relevant for the language server.
+        
+        This matcher is useful for operations outside of the language server,
+        such as when searching for relevant non-language files in the project.
+        """
+        return self._ignore_spec
 
     def should_ignore_path(self, relative_path: str) -> bool:
         """
@@ -288,7 +300,7 @@ class LanguageServer:
             normalized_path = normalized_path + '/'
         
         # Use the pathspec matcher to check if the path matches any ignore pattern
-        if self.ignore_spec.match_file(normalized_path):
+        if self._ignore_spec.match_file(normalized_path):
             return True
         
         return False
@@ -2008,3 +2020,15 @@ class SyncLanguageServer:
         Whether the given path should be ignored.
         """
         return self.language_server.should_ignore_path(relative_path)
+    
+    def get_ignore_spec(self) -> pathspec.PathSpec:
+        """Returns the pathspec matcher for the paths that were configured to be ignored through
+        the multilspy config file and the .gitignore file.
+        
+        This is is a subset of the full language-specific ignore spec that determines
+        which files are relevant for the language server.
+        
+        This matcher is useful for operations outside of the language server,
+        such as when searching for relevant non-language files in the project.
+        """
+        return self.language_server.get_ignore_spec()

src/serena/agent.py

@@ -26,6 +26,7 @@ from serena import serena_root_path, serena_version
 from serena.gui_log_viewer import GuiLogViewer, GuiLogViewerHandler
 from serena.llm.prompt_factory import PromptFactory
 from serena.symbol import SymbolLocation, SymbolManager
+from serena.text_utils import search_files
 from serena.util.class_decorators import singleton
 from serena.util.file_system import scan_directory
 from serena.util.inspection import iter_subclasses
@@ -1145,9 +1146,9 @@ class PrepareForNewConversationTool(Tool):
         return self.prompt_factory.create_prepare_for_new_conversation()
 
 
-class SearchInAllCodeTool(Tool):
+class SearchForPatternTool(Tool):
     """
-    Performs a search for a pattern in all code files (and only in code files) in the project.
+    Performs a search for a pattern in the project.
     """
 
     def apply(
@@ -1157,32 +1158,56 @@ class SearchInAllCodeTool(Tool):
         context_lines_after: int = 0,
         paths_include_glob: str | None = None,
         paths_exclude_glob: str | None = None,
+        only_in_code_files: bool = True,
         max_answer_chars: int = _DEFAULT_MAX_ANSWER_LENGTH,
     ) -> str:
         """
-        Search for a pattern in all code files (and only in code files) in the project. Generally, symbolic operations like find_symbol or find_referencing_symbols
+        Search for a pattern in the project. You can select whether all files or only code files should be searched.
+        Generally, symbolic operations like find_symbol or find_referencing_symbols
         should be preferred if you know which symbols you are looking for.
-        If you have to look in non-code files (like notebooks, documentation, etc.), you should use the shell_command tool with grep or similar.
-        This tool can be useful if you are looking for a specific pattern in the codebase that is not a symbol name.
 
         :param pattern: Regular expression pattern to search for, either as a compiled Pattern or string
         :param context_lines_before: Number of lines of context to include before each match
         :param context_lines_after: Number of lines of context to include after each match
         :param paths_include_glob: optional glob pattern specifying files to include in the search; if not provided, search globally.
         :param paths_exclude_glob: optional glob pattern specifying files to exclude from the search (takes precedence over paths_include_glob).
+        :param only_in_code_files: whether to search only in code files or in the entire code base.
+            The explicitly ignored files (from serena config and gitignore) are never searched.
         :param max_answer_chars: if the output is longer than this number of characters,
             no content will be returned. Don't adjust unless there is really no other way to get the content
             required for the task. Instead, if the output is too long, you should
             make a stricter query.
         :return: A JSON object mapping file paths to lists of matched consecutive lines (with context, if requested).
         """
-        matches = self.language_server.search_files_for_pattern(
-            pattern=pattern,
-            context_lines_before=context_lines_before,
-            context_lines_after=context_lines_after,
-            paths_include_glob=paths_include_glob,
-            paths_exclude_glob=paths_exclude_glob,
-        )
+        if only_in_code_files:
+            matches = self.language_server.search_files_for_pattern(
+                pattern=pattern,
+                context_lines_before=context_lines_before,
+                context_lines_after=context_lines_after,
+                paths_include_glob=paths_include_glob,
+                paths_exclude_glob=paths_exclude_glob,
+            )
+        else:
+            # we walk through all files in the project starting from the root
+            files_to_search = []
+            ignore_spec = self.language_server.get_ignore_spec()
+            for root, dirs, files in os.walk(self.project_root):
+                # Don't go into directories that are ignored by modifying dirs inplace
+                # Explanation for the  + "/" part:
+                # pathspec can't handle the matching of directories if they don't end with a slash!
+                # see https://github.com/cpburnz/python-pathspec/issues/89
+                dirs[:] = [d for d in dirs if not ignore_spec.match_file(d + "/")]
+                for file in files:
+                    if not ignore_spec.match_file(os.path.join(root, file)):
+                        files_to_search.append(os.path.join(root, file))
+            # TODO (maybe): not super efficient to walk through the files again and filter if glob patterns are provided
+            #   but it probably never matters and this version required no further refactoring
+            matches = search_files(
+                files_to_search,
+                pattern,
+                paths_include_glob=paths_include_glob,
+                paths_exclude_glob=paths_exclude_glob,
+            )
         # group matches by file
         file_to_matches: dict[str, list[str]] = defaultdict(list)
         for match in matches:

src/serena/text_utils.py

@@ -236,8 +236,12 @@ def search_files(
         if paths_exclude_glob and fnmatch(path, paths_exclude_glob):
             log.debug(f"Skipping {path}: matches exclude pattern {paths_exclude_glob}")
             continue
+        try:
+            file_content = content_reader(path)
+        except Exception as e:
+            log.error(f"Error reading file {path}. Skipping.\nError: {e}")
+            continue
 
-        file_content = content_reader(path)
         search_results = search_text(
             pattern,
             file_content,