From 739a4169438881b73b9cde3f327d3bc9c6d9bb8a Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Tue, 25 Mar 2025 20:18:52 +0100 Subject: [PATCH] LS: fixed request_parsed_files, added search_files_for_pattern --- src/multilspy/language_server.py | 110 ++++++++++++++++++++++++++++--- src/serena/text_utils.py | 4 +- test/multilspy/test_basic.py | 45 +++++++++++++ 3 files changed, 149 insertions(+), 10 deletions(-) diff --git a/src/multilspy/language_server.py b/src/multilspy/language_server.py index 0ce11474..c80720e8 100644 --- a/src/multilspy/language_server.py +++ b/src/multilspy/language_server.py @@ -8,16 +8,18 @@ The details of Language Specific configuration are not exposed to the user. import asyncio from copy import copy import dataclasses +from fnmatch import fnmatch import hashlib import json import pickle import logging import os import pathlib +import re import threading from contextlib import asynccontextmanager, contextmanager -from serena.text_utils import MatchedConsecutiveLines, LineType, TextLine +from serena.text_utils import MatchedConsecutiveLines, LineType, TextLine, search_text from .lsp_protocol_handler.lsp_constants import LSPConstants from .lsp_protocol_handler import lsp_types as LSPTypes @@ -102,11 +104,11 @@ class LanguageServer: return PyrightServer(config, logger, repository_root_path) # It used to be jedi, but pyright is a bit faster, and also more actively maintained # Keeping the previous code for reference - # from multilspy.language_servers.jedi_language_server.jedi_server import ( - # JediServer, - # ) + from multilspy.language_servers.jedi_language_server.jedi_server import ( + JediServer, + ) - # return JediServer(config, logger, repository_root_path) + return JediServer(config, logger, repository_root_path) elif config.code_language == Language.JAVA: from multilspy.language_servers.eclipse_jdtls.eclipse_jdtls import ( EclipseJDTLS, @@ -888,11 +890,79 @@ class LanguageServer: logging.ERROR, ) raise MultilspyException("Language Server not started") + # TODO: this worked in jedi, but pyright and basedpyright return nothing... + # I don't know why + # params = LSPTypes.WorkspaceSymbolParams(query="") # Empty query returns all symbols + # symbols = await self.server.send.workspace_symbol(params) or [] - params = LSPTypes.WorkspaceSymbolParams(query="") # Empty query returns all symbols - symbols = await self.server.send.workspace_symbol(params) or [] - return list({str(Path(s["location"]["uri"].replace("file://", "")).resolve().relative_to(self.repository_root_path)) for s in symbols}) + # Thus, instead of calling all symbols, we hack this and use the symbol tree instead, which + # seems to work in all these language servers + # walk through all children recursively, find all symbols of type Module and collect their relative paths + roots = await self.request_full_symbol_tree() + paths = [] + def collect_module_files(symbol): + if symbol["kind"] == multilspy_types.SymbolKind.Module: + assert "location" in symbol + paths.append(symbol["location"]["relativePath"]) + + elif symbol["kind"] == multilspy_types.SymbolKind.Package: + for child in symbol["children"]: + collect_module_files(child) + + for root in roots: + collect_module_files(root) + + return paths + + async def search_files_for_pattern( + self, + pattern: re.Pattern | str, + context_lines_before: int = 0, + context_lines_after: int = 0, + paths_include_glob: str | None = None, + paths_exclude_glob: str | None = None, + ) -> list[MatchedConsecutiveLines]: + """ + Search for a pattern across all files analyzed by the Language Server. + + :param pattern: Regular expression pattern to search for, either as a compiled Pattern or string + :param context_lines_before: Number of lines of context to include before each match + :param context_lines_after: Number of lines of context to include after each match + :param paths_include_glob: Glob pattern to filter which files to include in the search + :param paths_exclude_glob: Glob pattern to filter which files to exclude from the search. Takes precedence over paths_include_glob. + :return: List of matched consecutive lines with context + """ + if isinstance(pattern, str): + pattern = re.compile(pattern) + + matches = [] + all_files = await self.request_parsed_files() + for path in all_files: + # Apply glob filters if provided + if paths_include_glob and not fnmatch(path, paths_include_glob): + self.logger.log(f"Skipping {path}: does not match include pattern {paths_include_glob}", logging.DEBUG) + continue + + if paths_exclude_glob and fnmatch(path, paths_exclude_glob): + self.logger.log(f"Skipping {path}: matches exclude pattern {paths_exclude_glob}", logging.DEBUG) + continue + + file_content = self.retrieve_full_file_content(path) + search_results = search_text( + pattern, + file_content, + source_file_path=path, + allow_multiline_match=True, + context_lines_before=context_lines_before, + context_lines_after=context_lines_after + ) + if len(search_results) > 0: + self.logger.log(f"Found {len(search_results)} matches in {path}", logging.DEBUG) + matches.extend(search_results) + + return matches + async def request_referencing_symbols( self, relative_file_path: str, @@ -1555,6 +1625,30 @@ class SyncLanguageServer: """ return self.language_server.retrieve_content_around_line(relative_file_path, line, context_lines_before, context_lines_after) + def search_files_for_pattern( + self, + pattern: re.Pattern | str, + context_lines_before: int = 0, + context_lines_after: int = 0, + paths_include_glob: str | None = None, + paths_exclude_glob: str | None = None, + ) -> list[MatchedConsecutiveLines]: + """ + Search for a pattern across all files analyzed by the Language Server. + + :param pattern: Regular expression pattern to search for, either as a compiled Pattern or string + :param context_lines_before: Number of lines of context to include before each match + :param context_lines_after: Number of lines of context to include after each match + :param paths_include_glob: Glob pattern to filter which files to include in the search + :param paths_exclude_glob: Glob pattern to filter which files to exclude from the search. Takes precedence over paths_include_glob. + :return: List of matched consecutive lines with context + """ + assert self.loop + result = asyncio.run_coroutine_threadsafe( + self.language_server.search_files_for_pattern(pattern, context_lines_before, context_lines_after, paths_include_glob, paths_exclude_glob), self.loop + ).result() + return result + def start(self) -> "SyncLanguageServer": """ Starts the language server process and connects to it. Call shutdown when ready. diff --git a/src/serena/text_utils.py b/src/serena/text_utils.py index 70fcd5d1..4c1e1463 100644 --- a/src/serena/text_utils.py +++ b/src/serena/text_utils.py @@ -108,11 +108,11 @@ def search_text( :raises: ValueError if the pattern is not valid """ - if source_file_path and not content: + if source_file_path and content is None: with open(source_file_path) as f: content = f.read() - if not content: + if content is None: raise ValueError("Pass either content or source_file_path") matches = [] diff --git a/test/multilspy/test_basic.py b/test/multilspy/test_basic.py index c6b300d5..78df9341 100644 --- a/test/multilspy/test_basic.py +++ b/test/multilspy/test_basic.py @@ -165,3 +165,48 @@ class TestLanguageServerBasics: assert line.match_type == LineType.BEFORE_MATCH else: assert line.match_type == LineType.AFTER_MATCH + + def test_search_files_for_pattern(self, language_server: SyncLanguageServer, repo_path: Path): + """Test search_files_for_pattern with various patterns and glob filters.""" + # Test 1: Search for class definitions across all files + class_pattern = r"class\s+\w+\s*(?:\([^{]*\)|:)" + matches = language_server.search_files_for_pattern(class_pattern) + assert len(matches) > 0 + # Should find multiple classes like User, Item, BaseModel, etc. + assert len(matches) >= 5 + + # Test 2: Search for specific class with include glob + user_class_pattern = r"class\s+User\s*(?:\([^{]*\)|:)" + matches = language_server.search_files_for_pattern(user_class_pattern, paths_include_glob="**/models.py") + assert len(matches) == 1 # Should only find User class in models.py + assert matches[0].source_file_path is not None + assert "models.py" in matches[0].source_file_path + + # Test 3: Search for method definitions with exclude glob + method_pattern = r"def\s+\w+\s*\([^)]*\):" + matches = language_server.search_files_for_pattern(method_pattern, paths_exclude_glob="**/models.py") + assert len(matches) > 0 + # Should find methods in services.py but not in models.py + assert all(match.source_file_path is not None and "models.py" not in match.source_file_path for match in matches) + + # Test 4: Search for specific method with both include and exclude globs + create_user_pattern = r"def\s+create_user\s*\([^)]*\)(?:\s*->[^:]+)?:" + matches = language_server.search_files_for_pattern( + create_user_pattern, paths_include_glob="**/*.py", paths_exclude_glob="**/models.py" + ) + assert len(matches) == 1 # Should only find create_user in services.py + assert matches[0].source_file_path is not None + assert "services.py" in matches[0].source_file_path + + # Test 5: Search for a pattern that should appear in multiple files + init_pattern = r"def\s+__init__\s*\([^)]*\):" + matches = language_server.search_files_for_pattern(init_pattern) + assert len(matches) > 1 # Should find __init__ in multiple classes + # Should find __init__ in both models.py and services.py + assert any(match.source_file_path is not None and "models.py" in match.source_file_path for match in matches) + assert any(match.source_file_path is not None and "services.py" in match.source_file_path for match in matches) + + # Test 6: Search with a pattern that should have no matches + no_match_pattern = r"def\s+this_method_does_not_exist\s*\([^)]*\):" + matches = language_server.search_files_for_pattern(no_match_pattern) + assert len(matches) == 0