LS: fixed request_parsed_files, added search_files_for_pattern

src/multilspy/language_server.py

@@ -8,16 +8,18 @@ The details of Language Specific configuration are not exposed to the user.
 import asyncio
 from copy import copy
 import dataclasses
+from fnmatch import fnmatch
 import hashlib
 import json
 import pickle
 import logging
 import os
 import pathlib
+import re
 import threading
 from contextlib import asynccontextmanager, contextmanager
 
-from serena.text_utils import MatchedConsecutiveLines, LineType, TextLine
+from serena.text_utils import MatchedConsecutiveLines, LineType, TextLine, search_text
 from .lsp_protocol_handler.lsp_constants import LSPConstants
 from  .lsp_protocol_handler import lsp_types as LSPTypes
 
@@ -102,11 +104,11 @@ class LanguageServer:
             return PyrightServer(config, logger, repository_root_path)
             # It used to be jedi, but pyright is a bit faster, and also more actively maintained
             # Keeping the previous code for reference
-            # from multilspy.language_servers.jedi_language_server.jedi_server import (
-            #     JediServer,
-            # )
+            from multilspy.language_servers.jedi_language_server.jedi_server import (
+                JediServer,
+            )
 
-            # return JediServer(config, logger, repository_root_path)
+            return JediServer(config, logger, repository_root_path)
         elif config.code_language == Language.JAVA:
             from multilspy.language_servers.eclipse_jdtls.eclipse_jdtls import (
                 EclipseJDTLS,
@@ -888,11 +890,79 @@ class LanguageServer:
                 logging.ERROR,
             )
             raise MultilspyException("Language Server not started")
+        # TODO: this worked in jedi, but pyright and basedpyright return nothing...
+        # I don't know why
+        # params = LSPTypes.WorkspaceSymbolParams(query="")  # Empty query returns all symbols
+        # symbols = await self.server.send.workspace_symbol(params) or []
         
-        params = LSPTypes.WorkspaceSymbolParams(query="")  # Empty query returns all symbols
-        symbols = await self.server.send.workspace_symbol(params) or []
-        return list({str(Path(s["location"]["uri"].replace("file://", "")).resolve().relative_to(self.repository_root_path)) for s in symbols})
+        # Thus, instead of calling all symbols, we hack this and use the symbol tree instead, which
+        # seems to work in all these language servers
+        # walk through all children recursively, find all symbols of type Module and collect their relative paths
+        roots = await self.request_full_symbol_tree()
+        paths = [] 
+        def collect_module_files(symbol):
+            if symbol["kind"] == multilspy_types.SymbolKind.Module:
+                assert "location" in symbol
+                paths.append(symbol["location"]["relativePath"])
+            
+            elif symbol["kind"] == multilspy_types.SymbolKind.Package:
+                for child in symbol["children"]:
+                    collect_module_files(child)
+        
+        for root in roots:
+            collect_module_files(root)
+        
+        return paths
     
+    
+    async def search_files_for_pattern(
+        self,
+        pattern: re.Pattern | str,
+        context_lines_before: int = 0,
+        context_lines_after: int = 0,
+        paths_include_glob: str | None = None,
+        paths_exclude_glob: str | None = None,
+    ) -> list[MatchedConsecutiveLines]:
+        """
+        Search for a pattern across all files analyzed by the Language Server.
+        
+        :param pattern: Regular expression pattern to search for, either as a compiled Pattern or string
+        :param context_lines_before: Number of lines of context to include before each match
+        :param context_lines_after: Number of lines of context to include after each match
+        :param paths_include_glob: Glob pattern to filter which files to include in the search
+        :param paths_exclude_glob: Glob pattern to filter which files to exclude from the search. Takes precedence over paths_include_glob.
+        :return: List of matched consecutive lines with context
+        """
+        if isinstance(pattern, str):
+            pattern = re.compile(pattern)
+        
+        matches = []
+        all_files = await self.request_parsed_files()
+        for path in all_files:
+            # Apply glob filters if provided
+            if paths_include_glob and not fnmatch(path, paths_include_glob):
+                self.logger.log(f"Skipping {path}: does not match include pattern {paths_include_glob}", logging.DEBUG)
+                continue
+            
+            if paths_exclude_glob and fnmatch(path, paths_exclude_glob):
+                self.logger.log(f"Skipping {path}: matches exclude pattern {paths_exclude_glob}", logging.DEBUG)
+                continue
+                
+            file_content = self.retrieve_full_file_content(path)
+            search_results = search_text(
+                pattern, 
+                file_content, 
+                source_file_path=path, 
+                allow_multiline_match=True, 
+                context_lines_before=context_lines_before, 
+                context_lines_after=context_lines_after
+            )
+            if len(search_results) > 0:
+                self.logger.log(f"Found {len(search_results)} matches in {path}", logging.DEBUG)
+                matches.extend(search_results)
+        
+        return matches
+                
     async def request_referencing_symbols(
         self,
         relative_file_path: str,
@@ -1555,6 +1625,30 @@ class SyncLanguageServer:
         """
         return self.language_server.retrieve_content_around_line(relative_file_path, line, context_lines_before, context_lines_after)
     
+    def search_files_for_pattern(
+        self,
+        pattern: re.Pattern | str,
+        context_lines_before: int = 0,
+        context_lines_after: int = 0,
+        paths_include_glob: str | None = None,
+        paths_exclude_glob: str | None = None,
+    ) -> list[MatchedConsecutiveLines]:
+        """
+        Search for a pattern across all files analyzed by the Language Server.
+        
+        :param pattern: Regular expression pattern to search for, either as a compiled Pattern or string
+        :param context_lines_before: Number of lines of context to include before each match
+        :param context_lines_after: Number of lines of context to include after each match
+        :param paths_include_glob: Glob pattern to filter which files to include in the search
+        :param paths_exclude_glob: Glob pattern to filter which files to exclude from the search. Takes precedence over paths_include_glob.
+        :return: List of matched consecutive lines with context
+        """
+        assert self.loop
+        result = asyncio.run_coroutine_threadsafe(
+            self.language_server.search_files_for_pattern(pattern, context_lines_before, context_lines_after, paths_include_glob, paths_exclude_glob), self.loop
+        ).result()
+        return result
+    
     def start(self) -> "SyncLanguageServer":
         """
         Starts the language server process and connects to it. Call shutdown when ready.

src/serena/text_utils.py

@@ -108,11 +108,11 @@ def search_text(
     :raises: ValueError if the pattern is not valid
 
     """
-    if source_file_path and not content:
+    if source_file_path and content is None:
         with open(source_file_path) as f:
             content = f.read()
 
-    if not content:
+    if content is None:
         raise ValueError("Pass either content or source_file_path")
 
     matches = []

test/multilspy/test_basic.py

@@ -165,3 +165,48 @@ class TestLanguageServerBasics:
                 assert line.match_type == LineType.BEFORE_MATCH
             else:
                 assert line.match_type == LineType.AFTER_MATCH
+
+    def test_search_files_for_pattern(self, language_server: SyncLanguageServer, repo_path: Path):
+        """Test search_files_for_pattern with various patterns and glob filters."""
+        # Test 1: Search for class definitions across all files
+        class_pattern = r"class\s+\w+\s*(?:\([^{]*\)|:)"
+        matches = language_server.search_files_for_pattern(class_pattern)
+        assert len(matches) > 0
+        # Should find multiple classes like User, Item, BaseModel, etc.
+        assert len(matches) >= 5
+
+        # Test 2: Search for specific class with include glob
+        user_class_pattern = r"class\s+User\s*(?:\([^{]*\)|:)"
+        matches = language_server.search_files_for_pattern(user_class_pattern, paths_include_glob="**/models.py")
+        assert len(matches) == 1  # Should only find User class in models.py
+        assert matches[0].source_file_path is not None
+        assert "models.py" in matches[0].source_file_path
+
+        # Test 3: Search for method definitions with exclude glob
+        method_pattern = r"def\s+\w+\s*\([^)]*\):"
+        matches = language_server.search_files_for_pattern(method_pattern, paths_exclude_glob="**/models.py")
+        assert len(matches) > 0
+        # Should find methods in services.py but not in models.py
+        assert all(match.source_file_path is not None and "models.py" not in match.source_file_path for match in matches)
+
+        # Test 4: Search for specific method with both include and exclude globs
+        create_user_pattern = r"def\s+create_user\s*\([^)]*\)(?:\s*->[^:]+)?:"
+        matches = language_server.search_files_for_pattern(
+            create_user_pattern, paths_include_glob="**/*.py", paths_exclude_glob="**/models.py"
+        )
+        assert len(matches) == 1  # Should only find create_user in services.py
+        assert matches[0].source_file_path is not None
+        assert "services.py" in matches[0].source_file_path
+
+        # Test 5: Search for a pattern that should appear in multiple files
+        init_pattern = r"def\s+__init__\s*\([^)]*\):"
+        matches = language_server.search_files_for_pattern(init_pattern)
+        assert len(matches) > 1  # Should find __init__ in multiple classes
+        # Should find __init__ in both models.py and services.py
+        assert any(match.source_file_path is not None and "models.py" in match.source_file_path for match in matches)
+        assert any(match.source_file_path is not None and "services.py" in match.source_file_path for match in matches)
+
+        # Test 6: Search with a pattern that should have no matches
+        no_match_pattern = r"def\s+this_method_does_not_exist\s*\([^)]*\):"
+        matches = language_server.search_files_for_pattern(no_match_pattern)
+        assert len(matches) == 0