From ad753d308c751c1c87b9f8c0dc86fb0683e7d05c Mon Sep 17 00:00:00 2001
From: Michael Panchenko <michael.panchenko@unternehmertum.de>
Date: Wed, 28 May 2025 17:53:11 +0200
Subject: [PATCH] Massively sped up search_files by parallelizing reading of
 files

Used multithreading for executing file_reader
---
 src/serena/symbol.py     |  2 +-
 src/serena/text_utils.py | 35 +++++++++++++++++++----------------
 2 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/src/serena/symbol.py b/src/serena/symbol.py
index 8f3ab472..f426bfc9 100644
--- a/src/serena/symbol.py
+++ b/src/serena/symbol.py
@@ -155,7 +155,7 @@ class Symbol(ToStringMixin):
         else:
             # line is expected to be undefined for some types of symbols (e.g. SymbolKind.File)
             return None
-        
+
     @property
     def end_line(self) -> int | None:
         """The end line of the symbol body, also contained in the `body_end_position`."""
diff --git a/src/serena/text_utils.py b/src/serena/text_utils.py
index 41ec0dcc..aa0b2b77 100644
--- a/src/serena/text_utils.py
+++ b/src/serena/text_utils.py
@@ -3,8 +3,8 @@ import re
 from collections.abc import Callable
 from dataclasses import dataclass, field
 from enum import StrEnum
-from joblib import Parallel, delayed
 
+from joblib import Parallel, delayed
 from pathspec import PathSpec
 from pathspec.patterns.gitwildmatch import GitWildMatchPattern
 
@@ -104,7 +104,8 @@ def search_text(
         content: The text content to search. May be None if source_file_path is provided.
         source_file_path: Optional path to the source file. If content is None,
             this has to be passed and the file will be read.
-        allow_multiline_match: Whether to search across multiple lines
+        allow_multiline_match: Whether to search across multiple lines. Currently, the default
+            option (False) is very inefficient, so it is recommended to set this to True.
         context_lines_before: Number of context lines to include before matches
         context_lines_after: Number of context lines to include after matches
         is_glob: If True, pattern is treated as a glob-like pattern (e.g., "*.py", "test_??.py")
@@ -189,6 +190,8 @@ def search_text(
 
             matches.append(MatchedConsecutiveLines(lines=context_lines, source_file_path=source_file_path))
     else:
+        # TODO: extremely inefficient! Since we currently don't use this option in SerenaAgent or LanguageServer,
+        #   it is not urgent to fix, but should be either improved or the option should be removed.
         # Search line by line
         for i, line in enumerate(lines):
             line_num = i + 1
@@ -246,7 +249,7 @@ def search_files(
     # Pre-filter paths (done sequentially to avoid overhead)
     include_spec = PathSpec.from_lines(GitWildMatchPattern, [paths_include_glob]) if paths_include_glob else None
     exclude_spec = PathSpec.from_lines(GitWildMatchPattern, [paths_exclude_glob]) if paths_exclude_glob else None
-    
+
     filtered_paths = []
     for path in file_paths:
         if include_spec and not include_spec.match_file(path):
@@ -256,9 +259,9 @@ def search_files(
             log.debug(f"Skipping {path}: matches exclude pattern {paths_exclude_glob}")
             continue
         filtered_paths.append(path)
-    
+
     log.info(f"Processing {len(filtered_paths)} files.")
-    
+
     def process_single_file(path: str):
         """Process a single file - this function will be parallelized."""
         try:
@@ -273,29 +276,29 @@ def search_files(
             )
             if len(search_results) > 0:
                 log.debug(f"Found {len(search_results)} matches in {path}")
-            return {'path': path, 'results': search_results, 'error': None}
+            return {"path": path, "results": search_results, "error": None}
         except Exception as e:
             log.debug(f"Error processing {path}: {e}")
-            return {'path': path, 'results': [], 'error': str(e)}
-    
+            return {"path": path, "results": [], "error": str(e)}
+
     # Execute in parallel using joblib
     results = Parallel(
         n_jobs=-1,
         backend="threading",
     )(delayed(process_single_file)(path) for path in filtered_paths)
-    
+
     # Collect results and errors
     matches = []
     skipped_file_error_tuples = []
-    
+
     for result in results:
-        if result['error']:
-            skipped_file_error_tuples.append((result['path'], result['error']))
+        if result["error"]:
+            skipped_file_error_tuples.append((result["path"], result["error"]))
         else:
-            matches.extend(result['results'])
-    
+            matches.extend(result["results"])
+
     if skipped_file_error_tuples:
         log.debug(f"Failed to read {len(skipped_file_error_tuples)} files: {skipped_file_error_tuples}")
-    
+
     log.info(f"Found {len(matches)} total matches across {len(filtered_paths)} files")
-    return matches
\ No newline at end of file
+    return matches