From aaf2804a78a3417ef76af282fe483cbd18a05043 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Tue, 25 Mar 2025 19:11:29 +0100 Subject: [PATCH 1/3] Added exclude paths to pyright config --- src/multilspy/language_server.py | 7 +- .../pyright_language_server/pyright_server.py | 130 +++++++----------- 2 files changed, 56 insertions(+), 81 deletions(-) diff --git a/src/multilspy/language_server.py b/src/multilspy/language_server.py index 625dd467..0ce11474 100644 --- a/src/multilspy/language_server.py +++ b/src/multilspy/language_server.py @@ -876,7 +876,10 @@ class LanguageServer: async def request_parsed_files(self) -> list[str]: - """This is slow, as it finds all files by finding all symbols. + """ + Retrieves relative paths of all files analyzed by the Language Server. + + This is slow, as it finds all files by finding all symbols. This seems to be the only way, the LSP does not provide any endpoints for listing project files.""" if not self.server_started: @@ -888,7 +891,7 @@ class LanguageServer: params = LSPTypes.WorkspaceSymbolParams(query="") # Empty query returns all symbols symbols = await self.server.send.workspace_symbol(params) or [] - return list({s["location"]["uri"].replace("file://", "") for s in symbols}) + return list({str(Path(s["location"]["uri"].replace("file://", "")).resolve().relative_to(self.repository_root_path)) for s in symbols}) async def request_referencing_symbols( self, diff --git a/src/multilspy/language_servers/pyright_language_server/pyright_server.py b/src/multilspy/language_servers/pyright_language_server/pyright_server.py index 9ac00a64..60f7ada4 100644 --- a/src/multilspy/language_servers/pyright_language_server/pyright_server.py +++ b/src/multilspy/language_servers/pyright_language_server/pyright_server.py @@ -45,37 +45,33 @@ class PyrightServer(LanguageServer): "processId": os.getpid(), "rootPath": repository_absolute_path, "rootUri": pathlib.Path(repository_absolute_path).as_uri(), + "initializationOptions": { + "exclude": [ + "**/__pycache__", + "**/.venv", + "**/.env", + "**/build", + "**/dist", + "**/.pixi", + ], + "reportMissingImports": "error", + }, "capabilities": { "workspace": { "applyEdit": True, - "workspaceEdit": { - "documentChanges": True - }, - "didChangeConfiguration": { - "dynamicRegistration": True - }, - "didChangeWatchedFiles": { - "dynamicRegistration": True - }, + "workspaceEdit": {"documentChanges": True}, + "didChangeConfiguration": {"dynamicRegistration": True}, + "didChangeWatchedFiles": {"dynamicRegistration": True}, "symbol": { "dynamicRegistration": True, "symbolKind": { - "valueSet": [ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26 - ] - } + "valueSet": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26] + }, }, - "executeCommand": { - "dynamicRegistration": True - } + "executeCommand": {"dynamicRegistration": True}, }, "textDocument": { - "synchronization": { - "dynamicRegistration": True, - "willSave": True, - "willSaveWaitUntil": True, - "didSave": True - }, + "synchronization": {"dynamicRegistration": True, "willSave": True, "willSaveWaitUntil": True, "didSave": True}, "completion": { "dynamicRegistration": True, "contextSupport": True, @@ -84,84 +80,60 @@ class PyrightServer(LanguageServer): "commitCharactersSupport": True, "documentationFormat": ["markdown", "plaintext"], "deprecatedSupport": True, - "preselectSupport": True + "preselectSupport": True, }, "completionItemKind": { - "valueSet": [ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25 - ] - } - }, - "hover": { - "dynamicRegistration": True, - "contentFormat": ["markdown", "plaintext"] + "valueSet": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25] + }, }, + "hover": {"dynamicRegistration": True, "contentFormat": ["markdown", "plaintext"]}, "signatureHelp": { "dynamicRegistration": True, "signatureInformation": { "documentationFormat": ["markdown", "plaintext"], - "parameterInformation": { - "labelOffsetSupport": True - } - } - }, - "definition": { - "dynamicRegistration": True - }, - "references": { - "dynamicRegistration": True - }, - "documentHighlight": { - "dynamicRegistration": True + "parameterInformation": {"labelOffsetSupport": True}, + }, }, + "definition": {"dynamicRegistration": True}, + "references": {"dynamicRegistration": True}, + "documentHighlight": {"dynamicRegistration": True}, "documentSymbol": { "dynamicRegistration": True, "symbolKind": { - "valueSet": [ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26 - ] + "valueSet": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26] }, - "hierarchicalDocumentSymbolSupport": True + "hierarchicalDocumentSymbolSupport": True, }, "codeAction": { "dynamicRegistration": True, "codeActionLiteralSupport": { "codeActionKind": { "valueSet": [ - "", "quickfix", "refactor", "refactor.extract", "refactor.inline", - "refactor.rewrite", "source", "source.organizeImports" + "", + "quickfix", + "refactor", + "refactor.extract", + "refactor.inline", + "refactor.rewrite", + "source", + "source.organizeImports", ] } - } + }, }, - "codeLens": { - "dynamicRegistration": True - }, - "formatting": { - "dynamicRegistration": True - }, - "rangeFormatting": { - "dynamicRegistration": True - }, - "onTypeFormatting": { - "dynamicRegistration": True - }, - "rename": { - "dynamicRegistration": True - }, - "publishDiagnostics": { - "relatedInformation": True - } - } + "codeLens": {"dynamicRegistration": True}, + "formatting": {"dynamicRegistration": True}, + "rangeFormatting": {"dynamicRegistration": True}, + "onTypeFormatting": {"dynamicRegistration": True}, + "rename": {"dynamicRegistration": True}, + "publishDiagnostics": {"relatedInformation": True}, + }, }, "workspaceFolders": [ - { - "uri": pathlib.Path(repository_absolute_path).as_uri(), - "name": os.path.basename(repository_absolute_path) - } - ] + {"uri": pathlib.Path(repository_absolute_path).as_uri(), "name": os.path.basename(repository_absolute_path)} + ], } - + return initialize_params @asynccontextmanager @@ -205,16 +177,16 @@ class PyrightServer(LanguageServer): async with super().start_server(): self.logger.log("Starting pyright-langserver server process", logging.INFO) await self.server.start() - + # Send proper initialization parameters initialize_params = self._get_initialize_params(self.repository_root_path) - + self.logger.log( "Sending initialize request from LSP client to LSP server and awaiting response", logging.INFO, ) init_response = await self.server.send.initialize(initialize_params) - + # Verify that the server supports our required features self.server.notify.initialized({}) From 7e8caac90938645b03cc98c69da734aec6711cba Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Tue, 25 Mar 2025 20:18:13 +0100 Subject: [PATCH 2/3] Minor --- .vscode/settings.json | 3 ++- .../pyright_language_server/pyright_server.py | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 9e686107..d98e4514 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,6 @@ { "cSpell.words": [ - "multilspy" + "multilspy", + "pixi" ] } \ No newline at end of file diff --git a/src/multilspy/language_servers/pyright_language_server/pyright_server.py b/src/multilspy/language_servers/pyright_language_server/pyright_server.py index 60f7ada4..09e301f3 100644 --- a/src/multilspy/language_servers/pyright_language_server/pyright_server.py +++ b/src/multilspy/language_servers/pyright_language_server/pyright_server.py @@ -31,7 +31,8 @@ class PyrightServer(LanguageServer): config, logger, repository_root_path, - # we can also use `pyright-langserver --stdio` but it requires pyright to be installed with npm + # Note 1: we can also use `pyright-langserver --stdio` but it requires pyright to be installed with npm + # Note 2: we can also use `bpyright-langserver --stdio` if we ever are unhappy with pyright ProcessLaunchInfo(cmd="python -m pyright.langserver --stdio", cwd=repository_root_path), "python", ) @@ -41,7 +42,7 @@ class PyrightServer(LanguageServer): Returns the initialize params for the Pyright Language Server. """ # Create basic initialization parameters - initialize_params = { + initialize_params: InitializeParams = { # type: ignore "processId": os.getpid(), "rootPath": repository_absolute_path, "rootUri": pathlib.Path(repository_absolute_path).as_uri(), @@ -182,10 +183,11 @@ class PyrightServer(LanguageServer): initialize_params = self._get_initialize_params(self.repository_root_path) self.logger.log( - "Sending initialize request from LSP client to LSP server and awaiting response", + "Sending initialize request from LSP client to pyright server and awaiting response", logging.INFO, ) init_response = await self.server.send.initialize(initialize_params) + self.logger.log(f"Received initialize response from pyright server: {init_response}", logging.INFO) # Verify that the server supports our required features self.server.notify.initialized({}) From 739a4169438881b73b9cde3f327d3bc9c6d9bb8a Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Tue, 25 Mar 2025 20:18:52 +0100 Subject: [PATCH 3/3] LS: fixed request_parsed_files, added search_files_for_pattern --- src/multilspy/language_server.py | 110 ++++++++++++++++++++++++++++--- src/serena/text_utils.py | 4 +- test/multilspy/test_basic.py | 45 +++++++++++++ 3 files changed, 149 insertions(+), 10 deletions(-) diff --git a/src/multilspy/language_server.py b/src/multilspy/language_server.py index 0ce11474..c80720e8 100644 --- a/src/multilspy/language_server.py +++ b/src/multilspy/language_server.py @@ -8,16 +8,18 @@ The details of Language Specific configuration are not exposed to the user. import asyncio from copy import copy import dataclasses +from fnmatch import fnmatch import hashlib import json import pickle import logging import os import pathlib +import re import threading from contextlib import asynccontextmanager, contextmanager -from serena.text_utils import MatchedConsecutiveLines, LineType, TextLine +from serena.text_utils import MatchedConsecutiveLines, LineType, TextLine, search_text from .lsp_protocol_handler.lsp_constants import LSPConstants from .lsp_protocol_handler import lsp_types as LSPTypes @@ -102,11 +104,11 @@ class LanguageServer: return PyrightServer(config, logger, repository_root_path) # It used to be jedi, but pyright is a bit faster, and also more actively maintained # Keeping the previous code for reference - # from multilspy.language_servers.jedi_language_server.jedi_server import ( - # JediServer, - # ) + from multilspy.language_servers.jedi_language_server.jedi_server import ( + JediServer, + ) - # return JediServer(config, logger, repository_root_path) + return JediServer(config, logger, repository_root_path) elif config.code_language == Language.JAVA: from multilspy.language_servers.eclipse_jdtls.eclipse_jdtls import ( EclipseJDTLS, @@ -888,11 +890,79 @@ class LanguageServer: logging.ERROR, ) raise MultilspyException("Language Server not started") + # TODO: this worked in jedi, but pyright and basedpyright return nothing... + # I don't know why + # params = LSPTypes.WorkspaceSymbolParams(query="") # Empty query returns all symbols + # symbols = await self.server.send.workspace_symbol(params) or [] - params = LSPTypes.WorkspaceSymbolParams(query="") # Empty query returns all symbols - symbols = await self.server.send.workspace_symbol(params) or [] - return list({str(Path(s["location"]["uri"].replace("file://", "")).resolve().relative_to(self.repository_root_path)) for s in symbols}) + # Thus, instead of calling all symbols, we hack this and use the symbol tree instead, which + # seems to work in all these language servers + # walk through all children recursively, find all symbols of type Module and collect their relative paths + roots = await self.request_full_symbol_tree() + paths = [] + def collect_module_files(symbol): + if symbol["kind"] == multilspy_types.SymbolKind.Module: + assert "location" in symbol + paths.append(symbol["location"]["relativePath"]) + + elif symbol["kind"] == multilspy_types.SymbolKind.Package: + for child in symbol["children"]: + collect_module_files(child) + + for root in roots: + collect_module_files(root) + + return paths + + async def search_files_for_pattern( + self, + pattern: re.Pattern | str, + context_lines_before: int = 0, + context_lines_after: int = 0, + paths_include_glob: str | None = None, + paths_exclude_glob: str | None = None, + ) -> list[MatchedConsecutiveLines]: + """ + Search for a pattern across all files analyzed by the Language Server. + + :param pattern: Regular expression pattern to search for, either as a compiled Pattern or string + :param context_lines_before: Number of lines of context to include before each match + :param context_lines_after: Number of lines of context to include after each match + :param paths_include_glob: Glob pattern to filter which files to include in the search + :param paths_exclude_glob: Glob pattern to filter which files to exclude from the search. Takes precedence over paths_include_glob. + :return: List of matched consecutive lines with context + """ + if isinstance(pattern, str): + pattern = re.compile(pattern) + + matches = [] + all_files = await self.request_parsed_files() + for path in all_files: + # Apply glob filters if provided + if paths_include_glob and not fnmatch(path, paths_include_glob): + self.logger.log(f"Skipping {path}: does not match include pattern {paths_include_glob}", logging.DEBUG) + continue + + if paths_exclude_glob and fnmatch(path, paths_exclude_glob): + self.logger.log(f"Skipping {path}: matches exclude pattern {paths_exclude_glob}", logging.DEBUG) + continue + + file_content = self.retrieve_full_file_content(path) + search_results = search_text( + pattern, + file_content, + source_file_path=path, + allow_multiline_match=True, + context_lines_before=context_lines_before, + context_lines_after=context_lines_after + ) + if len(search_results) > 0: + self.logger.log(f"Found {len(search_results)} matches in {path}", logging.DEBUG) + matches.extend(search_results) + + return matches + async def request_referencing_symbols( self, relative_file_path: str, @@ -1555,6 +1625,30 @@ class SyncLanguageServer: """ return self.language_server.retrieve_content_around_line(relative_file_path, line, context_lines_before, context_lines_after) + def search_files_for_pattern( + self, + pattern: re.Pattern | str, + context_lines_before: int = 0, + context_lines_after: int = 0, + paths_include_glob: str | None = None, + paths_exclude_glob: str | None = None, + ) -> list[MatchedConsecutiveLines]: + """ + Search for a pattern across all files analyzed by the Language Server. + + :param pattern: Regular expression pattern to search for, either as a compiled Pattern or string + :param context_lines_before: Number of lines of context to include before each match + :param context_lines_after: Number of lines of context to include after each match + :param paths_include_glob: Glob pattern to filter which files to include in the search + :param paths_exclude_glob: Glob pattern to filter which files to exclude from the search. Takes precedence over paths_include_glob. + :return: List of matched consecutive lines with context + """ + assert self.loop + result = asyncio.run_coroutine_threadsafe( + self.language_server.search_files_for_pattern(pattern, context_lines_before, context_lines_after, paths_include_glob, paths_exclude_glob), self.loop + ).result() + return result + def start(self) -> "SyncLanguageServer": """ Starts the language server process and connects to it. Call shutdown when ready. diff --git a/src/serena/text_utils.py b/src/serena/text_utils.py index 70fcd5d1..4c1e1463 100644 --- a/src/serena/text_utils.py +++ b/src/serena/text_utils.py @@ -108,11 +108,11 @@ def search_text( :raises: ValueError if the pattern is not valid """ - if source_file_path and not content: + if source_file_path and content is None: with open(source_file_path) as f: content = f.read() - if not content: + if content is None: raise ValueError("Pass either content or source_file_path") matches = [] diff --git a/test/multilspy/test_basic.py b/test/multilspy/test_basic.py index c6b300d5..78df9341 100644 --- a/test/multilspy/test_basic.py +++ b/test/multilspy/test_basic.py @@ -165,3 +165,48 @@ class TestLanguageServerBasics: assert line.match_type == LineType.BEFORE_MATCH else: assert line.match_type == LineType.AFTER_MATCH + + def test_search_files_for_pattern(self, language_server: SyncLanguageServer, repo_path: Path): + """Test search_files_for_pattern with various patterns and glob filters.""" + # Test 1: Search for class definitions across all files + class_pattern = r"class\s+\w+\s*(?:\([^{]*\)|:)" + matches = language_server.search_files_for_pattern(class_pattern) + assert len(matches) > 0 + # Should find multiple classes like User, Item, BaseModel, etc. + assert len(matches) >= 5 + + # Test 2: Search for specific class with include glob + user_class_pattern = r"class\s+User\s*(?:\([^{]*\)|:)" + matches = language_server.search_files_for_pattern(user_class_pattern, paths_include_glob="**/models.py") + assert len(matches) == 1 # Should only find User class in models.py + assert matches[0].source_file_path is not None + assert "models.py" in matches[0].source_file_path + + # Test 3: Search for method definitions with exclude glob + method_pattern = r"def\s+\w+\s*\([^)]*\):" + matches = language_server.search_files_for_pattern(method_pattern, paths_exclude_glob="**/models.py") + assert len(matches) > 0 + # Should find methods in services.py but not in models.py + assert all(match.source_file_path is not None and "models.py" not in match.source_file_path for match in matches) + + # Test 4: Search for specific method with both include and exclude globs + create_user_pattern = r"def\s+create_user\s*\([^)]*\)(?:\s*->[^:]+)?:" + matches = language_server.search_files_for_pattern( + create_user_pattern, paths_include_glob="**/*.py", paths_exclude_glob="**/models.py" + ) + assert len(matches) == 1 # Should only find create_user in services.py + assert matches[0].source_file_path is not None + assert "services.py" in matches[0].source_file_path + + # Test 5: Search for a pattern that should appear in multiple files + init_pattern = r"def\s+__init__\s*\([^)]*\):" + matches = language_server.search_files_for_pattern(init_pattern) + assert len(matches) > 1 # Should find __init__ in multiple classes + # Should find __init__ in both models.py and services.py + assert any(match.source_file_path is not None and "models.py" in match.source_file_path for match in matches) + assert any(match.source_file_path is not None and "services.py" in match.source_file_path for match in matches) + + # Test 6: Search with a pattern that should have no matches + no_match_pattern = r"def\s+this_method_does_not_exist\s*\([^)]*\):" + matches = language_server.search_files_for_pattern(no_match_pattern) + assert len(matches) == 0