diff --git a/README.md b/README.md index f6caedbb..b8e906fc 100644 --- a/README.md +++ b/README.md @@ -597,7 +597,7 @@ Here is the full list of Serena's tools with a short description (output of `uv * `replace_lines`: Replaces a range of lines within a file with new content. * `replace_symbol_body`: Replaces the full definition of a symbol. * `restart_language_server`: Restarts the language server, may be necessary when edits not through Serena happen. -* `search_in_all_code`: Performs a search for a pattern in all code files (and only in code files) in the project. +* `search_for_pattern`: Performs a search for a pattern in the project. * `summarize_changes`: Provides instructions for summarizing the changes made to the codebase. * `think_about_collected_information`: Thinking tool for pondering the completeness of collected information. * `think_about_task_adherence`: Thinking tool for determining whether the agent is still on track with the current task. diff --git a/src/multilspy/language_server.py b/src/multilspy/language_server.py index a9d541d7..820c1a50 100644 --- a/src/multilspy/language_server.py +++ b/src/multilspy/language_server.py @@ -244,10 +244,22 @@ class LanguageServer: processed_patterns.append(line.strip()) # Create a pathspec matcher from the processed patterns - self.ignore_spec = pathspec.PathSpec.from_lines( + self._ignore_spec = pathspec.PathSpec.from_lines( pathspec.patterns.GitWildMatchPattern, processed_patterns ) + + def get_ignore_spec(self) -> pathspec.PathSpec: + """Returns the pathspec matcher for the paths that were configured to be ignored through + the multilspy config file and the .gitignore file. + + This is is a subset of the full language-specific ignore spec that determines + which files are relevant for the language server. + + This matcher is useful for operations outside of the language server, + such as when searching for relevant non-language files in the project. + """ + return self._ignore_spec def should_ignore_path(self, relative_path: str) -> bool: """ @@ -288,7 +300,7 @@ class LanguageServer: normalized_path = normalized_path + '/' # Use the pathspec matcher to check if the path matches any ignore pattern - if self.ignore_spec.match_file(normalized_path): + if self._ignore_spec.match_file(normalized_path): return True return False @@ -2008,3 +2020,15 @@ class SyncLanguageServer: Whether the given path should be ignored. """ return self.language_server.should_ignore_path(relative_path) + + def get_ignore_spec(self) -> pathspec.PathSpec: + """Returns the pathspec matcher for the paths that were configured to be ignored through + the multilspy config file and the .gitignore file. + + This is is a subset of the full language-specific ignore spec that determines + which files are relevant for the language server. + + This matcher is useful for operations outside of the language server, + such as when searching for relevant non-language files in the project. + """ + return self.language_server.get_ignore_spec() diff --git a/src/serena/agent.py b/src/serena/agent.py index 19a0ab56..d8a1f6d3 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -26,6 +26,7 @@ from serena import serena_root_path, serena_version from serena.gui_log_viewer import GuiLogViewer, GuiLogViewerHandler from serena.llm.prompt_factory import PromptFactory from serena.symbol import SymbolLocation, SymbolManager +from serena.text_utils import search_files from serena.util.class_decorators import singleton from serena.util.file_system import scan_directory from serena.util.inspection import iter_subclasses @@ -1145,9 +1146,9 @@ class PrepareForNewConversationTool(Tool): return self.prompt_factory.create_prepare_for_new_conversation() -class SearchInAllCodeTool(Tool): +class SearchForPatternTool(Tool): """ - Performs a search for a pattern in all code files (and only in code files) in the project. + Performs a search for a pattern in the project. """ def apply( @@ -1157,32 +1158,56 @@ class SearchInAllCodeTool(Tool): context_lines_after: int = 0, paths_include_glob: str | None = None, paths_exclude_glob: str | None = None, + only_in_code_files: bool = True, max_answer_chars: int = _DEFAULT_MAX_ANSWER_LENGTH, ) -> str: """ - Search for a pattern in all code files (and only in code files) in the project. Generally, symbolic operations like find_symbol or find_referencing_symbols + Search for a pattern in the project. You can select whether all files or only code files should be searched. + Generally, symbolic operations like find_symbol or find_referencing_symbols should be preferred if you know which symbols you are looking for. - If you have to look in non-code files (like notebooks, documentation, etc.), you should use the shell_command tool with grep or similar. - This tool can be useful if you are looking for a specific pattern in the codebase that is not a symbol name. :param pattern: Regular expression pattern to search for, either as a compiled Pattern or string :param context_lines_before: Number of lines of context to include before each match :param context_lines_after: Number of lines of context to include after each match :param paths_include_glob: optional glob pattern specifying files to include in the search; if not provided, search globally. :param paths_exclude_glob: optional glob pattern specifying files to exclude from the search (takes precedence over paths_include_glob). + :param only_in_code_files: whether to search only in code files or in the entire code base. + The explicitly ignored files (from serena config and gitignore) are never searched. :param max_answer_chars: if the output is longer than this number of characters, no content will be returned. Don't adjust unless there is really no other way to get the content required for the task. Instead, if the output is too long, you should make a stricter query. :return: A JSON object mapping file paths to lists of matched consecutive lines (with context, if requested). """ - matches = self.language_server.search_files_for_pattern( - pattern=pattern, - context_lines_before=context_lines_before, - context_lines_after=context_lines_after, - paths_include_glob=paths_include_glob, - paths_exclude_glob=paths_exclude_glob, - ) + if only_in_code_files: + matches = self.language_server.search_files_for_pattern( + pattern=pattern, + context_lines_before=context_lines_before, + context_lines_after=context_lines_after, + paths_include_glob=paths_include_glob, + paths_exclude_glob=paths_exclude_glob, + ) + else: + # we walk through all files in the project starting from the root + files_to_search = [] + ignore_spec = self.language_server.get_ignore_spec() + for root, dirs, files in os.walk(self.project_root): + # Don't go into directories that are ignored by modifying dirs inplace + # Explanation for the + "/" part: + # pathspec can't handle the matching of directories if they don't end with a slash! + # see https://github.com/cpburnz/python-pathspec/issues/89 + dirs[:] = [d for d in dirs if not ignore_spec.match_file(d + "/")] + for file in files: + if not ignore_spec.match_file(os.path.join(root, file)): + files_to_search.append(os.path.join(root, file)) + # TODO (maybe): not super efficient to walk through the files again and filter if glob patterns are provided + # but it probably never matters and this version required no further refactoring + matches = search_files( + files_to_search, + pattern, + paths_include_glob=paths_include_glob, + paths_exclude_glob=paths_exclude_glob, + ) # group matches by file file_to_matches: dict[str, list[str]] = defaultdict(list) for match in matches: diff --git a/src/serena/text_utils.py b/src/serena/text_utils.py index 774b09a7..857a8267 100644 --- a/src/serena/text_utils.py +++ b/src/serena/text_utils.py @@ -236,8 +236,12 @@ def search_files( if paths_exclude_glob and fnmatch(path, paths_exclude_glob): log.debug(f"Skipping {path}: matches exclude pattern {paths_exclude_glob}") continue + try: + file_content = content_reader(path) + except Exception as e: + log.error(f"Error reading file {path}. Skipping.\nError: {e}") + continue - file_content = content_reader(path) search_results = search_text( pattern, file_content,