From 369d41b7f28a60613199de91b86e1b09bfab4c60 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sun, 9 Nov 2025 20:30:14 +0100 Subject: [PATCH 01/26] Fix typo --- src/solidlsp/ls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index 93575dbb..20d737bf 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -840,7 +840,7 @@ class SolidLanguageServer(ABC): :param relative_file_path: The relative path of the file that has the symbols :param include_body: whether to include the body of the symbols in the result. :return: A list of symbols in the file, and a list of root symbols that represent the tree structure of the symbols. - All symbols will have a location, a children, and a parent attribute, + All symbols will have a location, children, and a parent attribute, where the parent attribute is None for root symbols. Note that this is slightly different from the call to request_full_symbol_tree, where the parent attribute will be the file symbol which in turn may have a package symbol as parent. From 12290821a0e089e2c25eb787455c978c163d2209 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sun, 9 Nov 2025 20:30:44 +0100 Subject: [PATCH 02/26] When creating LS launch command from a list, quote args containing spaces --- src/solidlsp/ls_handler.py | 4 ++-- src/solidlsp/util/subprocess_util.py | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/solidlsp/ls_handler.py b/src/solidlsp/ls_handler.py index 7dd0189c..ce7b48ac 100644 --- a/src/solidlsp/ls_handler.py +++ b/src/solidlsp/ls_handler.py @@ -33,7 +33,7 @@ from solidlsp.lsp_protocol_handler.server import ( make_request, make_response, ) -from solidlsp.util.subprocess_util import subprocess_kwargs +from solidlsp.util.subprocess_util import quote_arg, subprocess_kwargs log = logging.getLogger(__name__) @@ -189,7 +189,7 @@ class SolidLanguageServerHandler: if not isinstance(cmd, str) and not is_windows: # Since we are using the shell, we need to convert the command list to a single string # on Linux/macOS - cmd = " ".join(cmd) + cmd = " ".join(map(quote_arg, cmd)) log.info("Starting language server process via command: %s", self.process_launch_info.cmd) kwargs = subprocess_kwargs() kwargs["start_new_session"] = self.start_independent_lsp_process diff --git a/src/solidlsp/util/subprocess_util.py b/src/solidlsp/util/subprocess_util.py index ac6a4e2b..718d7afa 100644 --- a/src/solidlsp/util/subprocess_util.py +++ b/src/solidlsp/util/subprocess_util.py @@ -11,3 +11,12 @@ def subprocess_kwargs(): if platform.system() == "Windows": kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW # type: ignore return kwargs + + +def quote_arg(arg: str) -> str: + """ + Adds quotes around an argument if it contains spaces. + """ + if " " not in arg: + return arg + return f'"{arg}"' From 3d1521798c8cd37329993ef26c001af9f9c3b277 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sun, 9 Nov 2025 20:32:19 +0100 Subject: [PATCH 03/26] Do not naively join arguments; use list instead (fully robust on Windows) Quoting of arguments containing spaces is now handled centrally Fixes #735 --- .../language_servers/eclipse_jdtls.py | 68 +++++++++---------- 1 file changed, 33 insertions(+), 35 deletions(-) diff --git a/src/solidlsp/language_servers/eclipse_jdtls.py b/src/solidlsp/language_servers/eclipse_jdtls.py index e1181902..cb4fba44 100644 --- a/src/solidlsp/language_servers/eclipse_jdtls.py +++ b/src/solidlsp/language_servers/eclipse_jdtls.py @@ -109,41 +109,39 @@ class EclipseJDTLS(SolidLanguageServer): # TODO: Add "self.runtime_dependency_paths.jre_home_path"/bin to $PATH as well proc_env = {"syntaxserver": "false", "JAVA_HOME": self.runtime_dependency_paths.jre_home_path} proc_cwd = repository_root_path - cmd = " ".join( - [ - jre_path, - "--add-modules=ALL-SYSTEM", - "--add-opens", - "java.base/java.util=ALL-UNNAMED", - "--add-opens", - "java.base/java.lang=ALL-UNNAMED", - "--add-opens", - "java.base/sun.nio.fs=ALL-UNNAMED", - "-Declipse.application=org.eclipse.jdt.ls.core.id1", - "-Dosgi.bundles.defaultStartLevel=4", - "-Declipse.product=org.eclipse.jdt.ls.core.product", - "-Djava.import.generatesMetadataFilesAtProjectRoot=false", - "-Dfile.encoding=utf8", - "-noverify", - "-XX:+UseParallelGC", - "-XX:GCTimeRatio=4", - "-XX:AdaptiveSizePolicyWeight=90", - "-Dsun.zip.disableMemoryMapping=true", - "-Djava.lsp.joinOnCompletion=true", - "-Xmx3G", - "-Xms100m", - "-Xlog:disable", - "-Dlog.level=ALL", - f'"-javaagent:{lombok_jar_path}"', - f'"-Djdt.core.sharedIndexLocation={shared_cache_location}"', - "-jar", - f'"{jdtls_launcher_jar}"', - "-configuration", - f'"{jdtls_config_path}"', - "-data", - f'"{data_dir}"', - ] - ) + cmd = [ + jre_path, + "--add-modules=ALL-SYSTEM", + "--add-opens", + "java.base/java.util=ALL-UNNAMED", + "--add-opens", + "java.base/java.lang=ALL-UNNAMED", + "--add-opens", + "java.base/sun.nio.fs=ALL-UNNAMED", + "-Declipse.application=org.eclipse.jdt.ls.core.id1", + "-Dosgi.bundles.defaultStartLevel=4", + "-Declipse.product=org.eclipse.jdt.ls.core.product", + "-Djava.import.generatesMetadataFilesAtProjectRoot=false", + "-Dfile.encoding=utf8", + "-noverify", + "-XX:+UseParallelGC", + "-XX:GCTimeRatio=4", + "-XX:AdaptiveSizePolicyWeight=90", + "-Dsun.zip.disableMemoryMapping=true", + "-Djava.lsp.joinOnCompletion=true", + "-Xmx3G", + "-Xms100m", + "-Xlog:disable", + "-Dlog.level=ALL", + f"-javaagent:{lombok_jar_path}", + f"-Djdt.core.sharedIndexLocation={shared_cache_location}", + "-jar", + f"{jdtls_launcher_jar}", + "-configuration", + f"{jdtls_config_path}", + "-data", + f"{data_dir}", + ] self.service_ready_event = threading.Event() self.intellicode_enable_command_available = threading.Event() From 6f68b4d32c9050514ac8d3106dd2db85ee025d32 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sun, 9 Nov 2025 21:48:21 +0100 Subject: [PATCH 04/26] UnifiedSymbolInformation: Add overload_idx to support overloads #515 Adapt SolidLanguageServer.request_document_symbols to fill overload_idx Introduce symbol cache versioning within the saved file, avoiding storage of old cache files. --- src/solidlsp/ls.py | 126 ++++++++++++++++++++----------------- src/solidlsp/ls_types.py | 8 +++ src/solidlsp/util/cache.py | 23 +++++++ 3 files changed, 98 insertions(+), 59 deletions(-) create mode 100644 src/solidlsp/util/cache.py diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index 20d737bf..e1859dd9 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -4,7 +4,6 @@ import json import logging import os import pathlib -import pickle import shutil import subprocess import threading @@ -31,13 +30,14 @@ from solidlsp.ls_utils import FileUtils, PathUtils, TextUtils from solidlsp.lsp_protocol_handler import lsp_types from solidlsp.lsp_protocol_handler import lsp_types as LSPTypes from solidlsp.lsp_protocol_handler.lsp_constants import LSPConstants -from solidlsp.lsp_protocol_handler.lsp_types import Definition, DefinitionParams, LocationLink, SymbolKind +from solidlsp.lsp_protocol_handler.lsp_types import Definition, DefinitionParams, LocationLink from solidlsp.lsp_protocol_handler.server import ( LSPError, ProcessLaunchInfo, StringDict, ) from solidlsp.settings import SolidLSPSettings +from solidlsp.util.cache import load_cache, save_cache GenericDocumentSymbol = Union[LSPTypes.DocumentSymbol, LSPTypes.SymbolInformation, ls_types.UnifiedSymbolInformation] @@ -85,6 +85,7 @@ class SolidLanguageServer(ABC): """ CACHE_FOLDER_NAME = "cache" + CACHE_VERSION = 2 # To be overridden and extended by subclasses def is_ignored_dirname(self, dirname: str) -> bool: @@ -830,12 +831,26 @@ class SolidLanguageServer(ABC): return [json.loads(json_repr) for json_repr in set(json.dumps(item, sort_keys=True) for item in completions_list)] + def _request_document_symbols(self, relative_file_path: str, include_body: bool = False) -> list[GenericDocumentSymbol]: + """ + Sends a [textDocument/documentSymbol](https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_documentSymbol) request to the Language Server + to find symbols in the given file. Wait for the response and return the result. + + :param relative_file_path: + :param include_body: + :return: + """ + self.logger.log(f"Requesting document symbols for {relative_file_path} from the Language Server", logging.DEBUG) + response = self.server.send.document_symbol( + {"textDocument": {"uri": pathlib.Path(os.path.join(self.repository_root_path, relative_file_path)).as_uri()}} + ) + return response + def request_document_symbols( self, relative_file_path: str, include_body: bool = False ) -> tuple[list[ls_types.UnifiedSymbolInformation], list[ls_types.UnifiedSymbolInformation]]: """ - Raise a [textDocument/documentSymbol](https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_documentSymbol) request to the Language Server - to find symbols in the given file. Wait for the response and return the result. + Retrieves the full list of symbols in the given file, along with the root symbols that represent the tree structure of the symbols. :param relative_file_path: The relative path of the file that has the symbols :param include_body: whether to include the body of the symbols in the result. @@ -846,6 +861,7 @@ class SolidLanguageServer(ABC): where the parent attribute will be the file symbol which in turn may have a package symbol as parent. If you need a symbol tree that contains file symbols as well, you should use `request_full_symbol_tree` instead. """ + # check if the desired result is cached # TODO: it's kinda dumb to not use the cache if include_body is False after include_body was True once # Should be fixed in the future, it's a small performance optimization cache_key = f"{relative_file_path}-{include_body}" @@ -862,25 +878,30 @@ class SolidLanguageServer(ABC): else: self.logger.log(f"No cache hit for symbols with {include_body=} in {relative_file_path}", logging.DEBUG) - self.logger.log(f"Requesting document symbols for {relative_file_path} from the Language Server", logging.DEBUG) - response = self.server.send.document_symbol( - {"textDocument": {"uri": pathlib.Path(os.path.join(self.repository_root_path, relative_file_path)).as_uri()}} - ) - if response is None: - self.logger.log( - f"Received None response from the Language Server for document symbols in {relative_file_path}. " - f"This means the language server can't understand this file (possibly due to syntax errors). It may also be due to a bug or misconfiguration of the LS. " - f"Returning empty list", - logging.WARNING, - ) - return [], [] - assert isinstance(response, list), f"Unexpected response from Language Server: {response}" + # request the symbols from the language server + response = self._request_document_symbols(relative_file_path, include_body=include_body) + if response is None: self.logger.log( - f"Received {len(response)} document symbols for {relative_file_path} from the Language Server", - logging.DEBUG, + f"Received None response from the Language Server for document symbols in {relative_file_path}. " + f"This means the language server can't understand this file (possibly due to syntax errors). It may also be due to a bug or misconfiguration of the LS. " + f"Returning empty list", + logging.WARNING, ) + return [], [] + assert isinstance(response, list), f"Unexpected response from Language Server: {response}" + self.logger.log( + f"Received {len(response)} document symbols for {relative_file_path} from the Language Server", + logging.DEBUG, + ) - def turn_item_into_symbol_with_children(item: GenericDocumentSymbol): + def turn_item_into_symbol_with_children(item: GenericDocumentSymbol) -> ls_types.UnifiedSymbolInformation: + """ + Ensures that the given item is augmented with location, children, and body (if requested), + such that it satisfies the UnifiedSymbolInformation structure. + + :param item: the item to augment + :return: the augmented item (same object as input, but with more attributes and cast to UnifiedSymbolInformation) + """ item = cast(ls_types.UnifiedSymbolInformation, item) absolute_path = os.path.join(self.repository_root_path, relative_file_path) @@ -912,49 +933,36 @@ class SolidLanguageServer(ABC): for child in children: child["parent"] = item item[LSPConstants.CHILDREN] = children + return item flat_all_symbol_list: list[ls_types.UnifiedSymbolInformation] = [] - root_nodes: list[ls_types.UnifiedSymbolInformation] = [] - for root_item in response: - if "range" not in root_item and "location" not in root_item: - if root_item["kind"] in [SymbolKind.File, SymbolKind.Module]: - ... - # mutation is more convenient than creating a new dict, - # so we cast and rename the var after the mutating call to turn_item_into_symbol_with_children - # which turned and item into a "symbol" - turn_item_into_symbol_with_children(root_item) - root_symbol = cast(ls_types.UnifiedSymbolInformation, root_item) - root_symbol["parent"] = None + def convert_nodes_with_common_parent(nodes: list[GenericDocumentSymbol], parent: ls_types.UnifiedSymbolInformation | None) -> None: + """ + Converts the given nodes into UnifiedSymbolInformation with proper parent-child relationships, + adding overload indices for symbols with the same name under the same parent. + """ + name_counts = defaultdict(lambda: 0) + for node in nodes: + flat_all_symbol_list.append(node) + usymbol_node = turn_item_into_symbol_with_children(node) + usymbol_node["overload_idx"] = name_counts[usymbol_node["name"]] + name_counts[usymbol_node["name"]] += 1 + usymbol_node["parent"] = parent + if usymbol_node["children"]: + convert_nodes_with_common_parent(usymbol_node["children"], usymbol_node) - root_nodes.append(root_symbol) - assert isinstance(root_symbol, dict) - assert LSPConstants.NAME in root_symbol - assert LSPConstants.KIND in root_symbol - - if LSPConstants.CHILDREN in root_symbol: - # TODO: l_tree should be a list of TreeRepr. Define the following function to return TreeRepr as well - - def visit_tree_nodes_and_build_tree_repr(node: GenericDocumentSymbol) -> list[ls_types.UnifiedSymbolInformation]: - node = cast(ls_types.UnifiedSymbolInformation, node) - l: list[ls_types.UnifiedSymbolInformation] = [] - turn_item_into_symbol_with_children(node) - assert LSPConstants.CHILDREN in node - children = node[LSPConstants.CHILDREN] - l.append(node) - for child in children: - l.extend(visit_tree_nodes_and_build_tree_repr(child)) - return l - - flat_all_symbol_list.extend(visit_tree_nodes_and_build_tree_repr(root_symbol)) - else: - flat_all_symbol_list.append(ls_types.UnifiedSymbolInformation(**root_symbol)) + root_nodes: list[ls_types.UnifiedSymbolInformation] = response + convert_nodes_with_common_parent(root_nodes, None) result = flat_all_symbol_list, root_nodes + + # update cache self.logger.log(f"Caching document symbols for {relative_file_path}", logging.DEBUG) with self._cache_lock: self._document_symbols_cache[cache_key] = (file_data.content_hash, result) self._cache_has_changed = True + return result def request_full_symbol_tree( @@ -1561,7 +1569,7 @@ class SolidLanguageServer(ABC): / self._solidlsp_settings.project_data_relative_path / self.CACHE_FOLDER_NAME / self.language_id - / "document_symbols_cache_v23-06-25.pkl" + / "document_symbols_cache.pkl" ) def save_cache(self): @@ -1573,8 +1581,7 @@ class SolidLanguageServer(ABC): self.logger.log(f"Saving updated document symbols cache to {self.cache_path}", logging.INFO) self.cache_path.parent.mkdir(parents=True, exist_ok=True) try: - with open(self.cache_path, "wb") as f: - pickle.dump(self._document_symbols_cache, f) + save_cache(str(self.cache_path), self.CACHE_VERSION, self._document_symbols_cache) self._cache_has_changed = False except Exception as e: self.logger.log( @@ -1590,9 +1597,10 @@ class SolidLanguageServer(ABC): with self._cache_lock: self.logger.log(f"Loading document symbols cache from {self.cache_path}", logging.INFO) try: - with open(self.cache_path, "rb") as f: - self._document_symbols_cache = pickle.load(f) - self.logger.log(f"Loaded {len(self._document_symbols_cache)} document symbols from cache.", logging.INFO) + saved_cache = load_cache(str(self.cache_path), self.CACHE_VERSION) + if saved_cache is not None: + self._document_symbols_cache = saved_cache + self.logger.log(f"Loaded {len(self._document_symbols_cache)} document symbols from cache.", logging.INFO) except Exception as e: # cache often becomes corrupt, so just skip loading it self.logger.log( diff --git a/src/solidlsp/ls_types.py b/src/solidlsp/ls_types.py index 4a8de6d4..8bc6f303 100644 --- a/src/solidlsp/ls_types.py +++ b/src/solidlsp/ls_types.py @@ -243,6 +243,14 @@ class UnifiedSymbolInformation(TypedDict): All symbols except the root packages will have a parent. """ + overload_idx: NotRequired[int] + """ + The overload index of the symbol, if applicable. If the symbol is an overloaded function or method + (same symbol name with the same parent), this index indicates which overload it is. + The index is 0-based. + Added for Serena, not part of the LSP. + """ + class MarkupKind(Enum): """Describes the content type that a client supports in various diff --git a/src/solidlsp/util/cache.py b/src/solidlsp/util/cache.py new file mode 100644 index 00000000..64ee1345 --- /dev/null +++ b/src/solidlsp/util/cache.py @@ -0,0 +1,23 @@ +import logging +from typing import Any, Optional + +from sensai.util.pickle import dump_pickle, load_pickle + +log = logging.getLogger(__name__) + + +def load_cache(path: str, version: Any) -> Optional[Any]: + data = load_pickle(path) + if not isinstance(data, dict) or "__cache_version" not in data: + log.info("Cache is outdated (expected version %s). Ignoring cache at %s", version, path) + return None + saved_version = data["__cache_version"] + if saved_version != version: + log.info("Cache is outdated (expected version %s, got %s). Ignoring cache at %s", version, saved_version, path) + return None + return data["obj"] + + +def save_cache(path: str, version: Any, obj: Any) -> None: + data = {"__cache_version": version, "obj": obj} + dump_pickle(data, path) From 81052f7e9f6862b8d53c50c518704d2667902660 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sun, 9 Nov 2025 22:11:04 +0100 Subject: [PATCH 05/26] EclipseJDTLS: Remove type information from names, e.g. converting 'myMethod(int) ' to just 'myMethod' handling overloads via SolidLSP's newly introduced overload_idx instead Fixes #578 --- .../language_servers/eclipse_jdtls.py | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/solidlsp/language_servers/eclipse_jdtls.py b/src/solidlsp/language_servers/eclipse_jdtls.py index cb4fba44..8ade5b70 100644 --- a/src/solidlsp/language_servers/eclipse_jdtls.py +++ b/src/solidlsp/language_servers/eclipse_jdtls.py @@ -13,7 +13,7 @@ from pathlib import PurePath from overrides import override -from solidlsp.ls import SolidLanguageServer +from solidlsp.ls import GenericDocumentSymbol, SolidLanguageServer from solidlsp.ls_config import LanguageServerConfig from solidlsp.ls_logger import LanguageServerLogger from solidlsp.ls_utils import FileUtils, PlatformUtils @@ -788,3 +788,23 @@ class EclipseJDTLS(SolidLanguageServer): # TODO: Add comments about why we wait here, and how this can be optimized self.service_ready_event.wait() + + def _request_document_symbols(self, relative_file_path: str, include_body: bool = False) -> list[GenericDocumentSymbol]: + result = super()._request_document_symbols(relative_file_path, include_body) + + # JDTLS sometimes returns symbol names with type information to handle overloads, + # e.g. "myMethod(int) ", but we want overloads to be handled via overload_idx, + # which requires the name to be just "myMethod". + + def fix_name(symbol: GenericDocumentSymbol): + if "(" in symbol["name"]: + symbol["name"] = symbol["name"][: symbol["name"].index("(")] + children = symbol.get("children") + if children: + for child in children: + fix_name(child) + + for root_symbol in result: + fix_name(root_symbol) + + return result From 55b4a665fd44789aced32d7929d17cc32c7007a3 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sun, 9 Nov 2025 22:19:19 +0100 Subject: [PATCH 06/26] UnifiedSymbolInformation: Set overload_idx only if there are overloads #515 --- src/solidlsp/ls.py | 6 +++++- src/solidlsp/ls_types.py | 6 +++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index e1859dd9..eded9417 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -942,11 +942,15 @@ class SolidLanguageServer(ABC): Converts the given nodes into UnifiedSymbolInformation with proper parent-child relationships, adding overload indices for symbols with the same name under the same parent. """ + total_name_counts = defaultdict(lambda: 0) + for node in nodes: + total_name_counts[node["name"]] += 1 name_counts = defaultdict(lambda: 0) for node in nodes: flat_all_symbol_list.append(node) usymbol_node = turn_item_into_symbol_with_children(node) - usymbol_node["overload_idx"] = name_counts[usymbol_node["name"]] + if total_name_counts[usymbol_node["name"]] > 1: + usymbol_node["overload_idx"] = name_counts[usymbol_node["name"]] name_counts[usymbol_node["name"]] += 1 usymbol_node["parent"] = parent if usymbol_node["children"]: diff --git a/src/solidlsp/ls_types.py b/src/solidlsp/ls_types.py index 8bc6f303..bbd3cbad 100644 --- a/src/solidlsp/ls_types.py +++ b/src/solidlsp/ls_types.py @@ -245,9 +245,9 @@ class UnifiedSymbolInformation(TypedDict): overload_idx: NotRequired[int] """ - The overload index of the symbol, if applicable. If the symbol is an overloaded function or method - (same symbol name with the same parent), this index indicates which overload it is. - The index is 0-based. + The overload index of the symbol, if applicable. If a symbol does not have overloads, this field is omitted. + If the symbol is an overloaded function or method (same symbol name with the same parent), + this index indicates which overload it is. The index is 0-based. Added for Serena, not part of the LSP. """ From d31cfab25d712aaa51fec2e8381b852658dde563 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sun, 9 Nov 2025 23:00:57 +0100 Subject: [PATCH 07/26] Add overload_idx to name path matching, reifying NamePathMatcher for efficency Implements #515 --- src/serena/symbol.py | 88 ++++++++++++++++++++++++-------------- test/serena/test_symbol.py | 22 ++++++++-- 2 files changed, 76 insertions(+), 34 deletions(-) diff --git a/src/serena/symbol.py b/src/serena/symbol.py index e97b6c77..589c0d20 100644 --- a/src/serena/symbol.py +++ b/src/serena/symbol.py @@ -19,6 +19,7 @@ if TYPE_CHECKING: from .agent import SerenaAgent log = logging.getLogger(__name__) +NAME_PATH_SEP = "/" @dataclass @@ -113,45 +114,65 @@ class Symbol(ToStringMixin, ABC): """ -class LanguageServerSymbol(Symbol, ToStringMixin): - _NAME_PATH_SEP = "/" - - @staticmethod - def match_name_path( - name_path: str, - symbol_name_path_parts: list[str], - substring_matching: bool, - ) -> bool: +class NamePathMatcher(ToStringMixin): + def __init__(self, name_path_expr: str, substring_matching: bool) -> None: """ - Checks if a given `name_path` matches a symbol's qualified name parts. - See docstring of `Symbol.find` for more details. + :param name_path_expr: the name path expression to match against + :param substring_matching: whether to use substring matching for the last segment """ - assert name_path, "name_path must not be empty" - assert symbol_name_path_parts, "symbol_name_path_parts must not be empty" - name_path_sep = LanguageServerSymbol._NAME_PATH_SEP + assert name_path_expr, "name_path must not be empty" + self._expr = name_path_expr + self._substring_matching = substring_matching + self._is_absolute_pattern = name_path_expr.startswith(NAME_PATH_SEP) + self._pattern_parts = name_path_expr.lstrip(NAME_PATH_SEP).rstrip(NAME_PATH_SEP).split(NAME_PATH_SEP) - is_absolute_pattern = name_path.startswith(name_path_sep) - pattern_parts = name_path.lstrip(name_path_sep).rstrip(name_path_sep).split(name_path_sep) + # extract overload index "[idx]" if present at end of last part + self._overload_idx: int | None = None + last_part = self._pattern_parts[-1] + if last_part.endswith("]") and "[" in last_part: + bracket_idx = last_part.rfind("[") + index_part = last_part[bracket_idx + 1 : -1] + if index_part.isdigit(): + self._pattern_parts[-1] = last_part[:bracket_idx] + self._overload_idx = int(index_part) + def _tostring_includes(self) -> list[str]: + return ["_expr"] + + def matches_ls_symbol(self, symbol: "LanguageServerSymbol") -> bool: + return self.matches_components(symbol.get_name_path_parts(), symbol.overload_idx) + + def matches_components(self, symbol_name_path_parts: list[str], overload_idx: int | None) -> bool: # filtering based on ancestors - if len(pattern_parts) > len(symbol_name_path_parts): + if len(self._pattern_parts) > len(symbol_name_path_parts): # can't possibly match if pattern has more parts than symbol return False - if is_absolute_pattern and len(pattern_parts) != len(symbol_name_path_parts): + if self._is_absolute_pattern and len(self._pattern_parts) != len(symbol_name_path_parts): # for absolute patterns, the number of parts must match exactly return False - if symbol_name_path_parts[-len(pattern_parts) : -1] != pattern_parts[:-1]: + if symbol_name_path_parts[-len(self._pattern_parts) : -1] != self._pattern_parts[:-1]: # ancestors must match return False # matching the last part of the symbol name - name_to_match = pattern_parts[-1] + name_to_match = self._pattern_parts[-1] symbol_name = symbol_name_path_parts[-1] - if substring_matching: - return name_to_match in symbol_name + if self._substring_matching: + if name_to_match not in symbol_name: + return False else: - return name_to_match == symbol_name + if name_to_match != symbol_name: + return False + # check for matching overload index + if self._overload_idx is not None: + if overload_idx != self._overload_idx: + return False + + return True + + +class LanguageServerSymbol(Symbol, ToStringMixin): def __init__(self, symbol_root_from_ls: UnifiedSymbolInformation) -> None: self.symbol_root = symbol_root_from_ls @@ -173,6 +194,10 @@ class LanguageServerSymbol(Symbol, ToStringMixin): def symbol_kind(self) -> SymbolKind: return self.symbol_root["kind"] + @property + def overload_idx(self) -> int | None: + return self.symbol_root.get("overload_idx") + def is_neighbouring_definition_separated_by_empty_line(self) -> bool: return self.symbol_kind in (SymbolKind.Function, SymbolKind.Method, SymbolKind.Class, SymbolKind.Interface, SymbolKind.Struct) @@ -256,9 +281,13 @@ class LanguageServerSymbol(Symbol, ToStringMixin): def get_name_path(self) -> str: """ - Get the name path of the symbol (e.g. "class/method/inner_function"). + Get the name path of the symbol, e.g. "class/method/inner_function" or + "class/method[1]" (overloaded method with identifying index). """ - return self._NAME_PATH_SEP.join(self.get_name_path_parts()) + name_path = NAME_PATH_SEP.join(self.get_name_path_parts()) + if "overload_idx" in self.symbol_root: + name_path += f"[{self.symbol_root['overload_idx']}]" + return name_path def get_name_path_parts(self) -> list[str]: """ @@ -321,7 +350,7 @@ class LanguageServerSymbol(Symbol, ToStringMixin): For example, passing `/class` will match only against top-level symbols like `class` but not against `nested_class/class`. Passing `/class/method` will match against `class/method` but not `nested_class/class/method` or `method`. - :param name_path: the name path to match against + :param name_path: the name path expression to match against :param substring_matching: whether to use substring matching (as opposed to exact matching) of the last segment of `name_path` against the symbol name. :param include_kinds: an optional sequence of ints representing the LSP symbol kind. @@ -329,17 +358,14 @@ class LanguageServerSymbol(Symbol, ToStringMixin): :param exclude_kinds: If provided, symbols of the given kinds will be excluded from the result. """ result = [] + name_path_matcher = NamePathMatcher(name_path, substring_matching) def should_include(s: "LanguageServerSymbol") -> bool: if include_kinds is not None and s.symbol_kind not in include_kinds: return False if exclude_kinds is not None and s.symbol_kind in exclude_kinds: return False - return LanguageServerSymbol.match_name_path( - name_path=name_path, - symbol_name_path_parts=s.get_name_path_parts(), - substring_matching=substring_matching, - ) + return name_path_matcher.matches_ls_symbol(s) def traverse(s: "LanguageServerSymbol") -> None: if should_include(s): diff --git a/test/serena/test_symbol.py b/test/serena/test_symbol.py index 179693f0..d7dc772d 100644 --- a/test/serena/test_symbol.py +++ b/test/serena/test_symbol.py @@ -1,6 +1,6 @@ import pytest -from src.serena.symbol import LanguageServerSymbol +from serena.symbol import NamePathMatcher class TestSymbolNameMatching: @@ -77,7 +77,7 @@ class TestSymbolNameMatching: ) def test_match_simple_name(self, name_path_pattern, symbol_name_path_parts, is_substring_match, expected): """Tests matching for simple names (no '/' in pattern).""" - result = LanguageServerSymbol.match_name_path(name_path_pattern, symbol_name_path_parts, is_substring_match) + result = NamePathMatcher(name_path_pattern, is_substring_match).matches_components(symbol_name_path_parts, None) error_msg = self._create_assertion_error_message(name_path_pattern, symbol_name_path_parts, is_substring_match, expected, result) assert result == expected, error_msg @@ -157,6 +157,22 @@ class TestSymbolNameMatching: ) def test_match_name_path_pattern_path_len_2(self, name_path_pattern, symbol_name_path_parts, is_substring_match, expected): """Tests matching for qualified names (e.g. 'module/class/func').""" - result = LanguageServerSymbol.match_name_path(name_path_pattern, symbol_name_path_parts, is_substring_match) + result = NamePathMatcher(name_path_pattern, is_substring_match).matches_components(symbol_name_path_parts, None) error_msg = self._create_assertion_error_message(name_path_pattern, symbol_name_path_parts, is_substring_match, expected, result) assert result == expected, error_msg + + @pytest.mark.parametrize( + "name_path_pattern, symbol_name_path_parts, symbol_overload_idx, expected", + [ + pytest.param("bar/foo", ["bar", "foo"], 0, True, id="R: 'bar/foo' matches ['bar', 'foo'] with overload_index=0"), + pytest.param("bar/foo", ["bar", "foo"], 1, True, id="R: 'bar/foo' matches ['bar', 'foo'] with overload_index=1"), + pytest.param("bar/foo[0]", ["bar", "foo"], 0, True, id="R: 'bar/foo[0]' matches ['bar', 'foo'] with overload_index=0"), + pytest.param("bar/foo[1]", ["bar", "foo"], 0, False, id="R: 'bar/foo[1]' does not match ['bar', 'foo'] with overload_index=0"), + ], + ) + def test_match_name_path_pattern_with_overload_idx(self, name_path_pattern, symbol_name_path_parts, symbol_overload_idx, expected): + """Tests matching for qualified names (e.g. 'module/class/func').""" + matcher = NamePathMatcher(name_path_pattern, False) + result = matcher.matches_components(symbol_name_path_parts, symbol_overload_idx) + error_msg = self._create_assertion_error_message(name_path_pattern, symbol_name_path_parts, False, expected, result) + assert result == expected, error_msg From 3eda628eb639b201f94bc5c95a1cbddfb156898d Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 10 Nov 2025 00:05:12 +0100 Subject: [PATCH 08/26] SolidLanguageServer.request_document_symbols: Ensure file is opened while issuing documentSymbols request --- src/solidlsp/ls.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index eded9417..5092cf1f 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -833,12 +833,14 @@ class SolidLanguageServer(ABC): def _request_document_symbols(self, relative_file_path: str, include_body: bool = False) -> list[GenericDocumentSymbol]: """ - Sends a [textDocument/documentSymbol](https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_documentSymbol) request to the Language Server - to find symbols in the given file. Wait for the response and return the result. + Sends a [documentSymbol](https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_documentSymbol) + request to the language server to find symbols in the given file, which is assumed to be opened while the call is active. + Waits for the response and returns it. - :param relative_file_path: - :param include_body: - :return: + :param relative_file_path: the relative path of the file that has the symbols. + IMPORTANT: It must be ensured that the file is already opened in the Language Server before calling this method. + :param include_body: whether to include the body of the symbols in the result. + :return: the list of symbols in the file. """ self.logger.log(f"Requesting document symbols for {relative_file_path} from the Language Server", logging.DEBUG) response = self.server.send.document_symbol( @@ -878,8 +880,9 @@ class SolidLanguageServer(ABC): else: self.logger.log(f"No cache hit for symbols with {include_body=} in {relative_file_path}", logging.DEBUG) - # request the symbols from the language server - response = self._request_document_symbols(relative_file_path, include_body=include_body) + # request the symbols from the language server + response = self._request_document_symbols(relative_file_path, include_body=include_body) + if response is None: self.logger.log( f"Received None response from the Language Server for document symbols in {relative_file_path}. " From 47cdb297edd80d1147154616e68da6b1f5ea2002 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 10 Nov 2025 17:01:33 +0100 Subject: [PATCH 09/26] Improve symbolic tool class docstrings to make the symbolic backend clear --- src/serena/tools/jetbrains_tools.py | 6 +++--- src/serena/tools/symbol_tools.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/serena/tools/jetbrains_tools.py b/src/serena/tools/jetbrains_tools.py index db3a97c7..93de446e 100644 --- a/src/serena/tools/jetbrains_tools.py +++ b/src/serena/tools/jetbrains_tools.py @@ -6,7 +6,7 @@ from serena.tools.jetbrains_plugin_client import JetBrainsPluginClient class JetBrainsFindSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): """ - Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type). + Performs a global (or local) search for symbols using the JetBrains backend """ def apply( @@ -73,7 +73,7 @@ class JetBrainsFindSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): """ - Finds symbols that reference the given symbol + Finds symbols that reference the given symbol using the JetBrains backend """ def apply( @@ -104,7 +104,7 @@ class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead, ToolMark class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): """ - Retrieves an overview of the top-level symbols within a specified file + Retrieves an overview of the top-level symbols within a specified file using the JetBrains backend """ def apply( diff --git a/src/serena/tools/symbol_tools.py b/src/serena/tools/symbol_tools.py index bd31fc48..bddb749c 100644 --- a/src/serena/tools/symbol_tools.py +++ b/src/serena/tools/symbol_tools.py @@ -79,7 +79,7 @@ class GetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead): class FindSymbolTool(Tool, ToolMarkerSymbolicRead): """ - Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type). + Performs a global (or local) search using the language server backend. """ # noinspection PyDefaultArgument @@ -163,7 +163,7 @@ class FindSymbolTool(Tool, ToolMarkerSymbolicRead): class FindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead): """ - Finds symbols that reference the symbol at the given location (optionally filtered by type). + Finds symbols that reference the given symbol using the language server backend """ # noinspection PyDefaultArgument @@ -216,7 +216,7 @@ class FindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead): class ReplaceSymbolBodyTool(Tool, ToolMarkerSymbolicEdit): """ - Replaces the full definition of a symbol. + Replaces the full definition of a symbol using the language server backend. """ def apply( From 375bc18e38650f095872fc4cc5d590fa01406bd8 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 10 Nov 2025 17:02:46 +0100 Subject: [PATCH 10/26] Simplify name path and name path matching pattern descriptions, properly separating the two concepts --- .../resources/config/contexts/chatgpt.yml | 4 +- src/serena/tools/jetbrains_tools.py | 50 +++++++---------- src/serena/tools/symbol_tools.py | 53 ++++++++----------- test/serena/test_serena_agent.py | 8 +-- 4 files changed, 46 insertions(+), 69 deletions(-) diff --git a/src/serena/resources/config/contexts/chatgpt.yml b/src/serena/resources/config/contexts/chatgpt.yml index c758b2c3..b3780f4c 100644 --- a/src/serena/resources/config/contexts/chatgpt.yml +++ b/src/serena/resources/config/contexts/chatgpt.yml @@ -16,8 +16,8 @@ included_optional_tools: tool_description_overrides: find_symbol: | - Retrieves symbols matching `name_path` in a file. - Use `depth > 0` to include children. `name_path` can be: "foo": any symbol named "foo"; "foo/bar": "bar" within "foo"; "/foo/bar": only top-level "foo/bar" + Retrieves symbols matching `name_path_pattern` in a file. + Use `depth > 0` to include children. `name_path_pattern` can be: "foo": any symbol named "foo"; "foo/bar": "bar" within "foo"; "/foo/bar": only top-level "foo/bar" replace_regex: | Replaces text using regular expressions. Preferred for smaller edits where symbol-level tools aren't appropriate. Use wildcards (.*?) to match large sections efficiently: "beginning.*?end" instead of specifying exact content. diff --git a/src/serena/tools/jetbrains_tools.py b/src/serena/tools/jetbrains_tools.py index 93de446e..97cfd19f 100644 --- a/src/serena/tools/jetbrains_tools.py +++ b/src/serena/tools/jetbrains_tools.py @@ -11,46 +11,34 @@ class JetBrainsFindSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): def apply( self, - name_path: str, + name_path_pattern: str, depth: int = 0, relative_path: str | None = None, include_body: bool = False, max_answer_chars: int = -1, ) -> str: """ - Retrieves information on all symbols/code entities (classes, methods, etc.) based on the given `name_path`, - which represents a pattern for the symbol's path within the symbol tree of a single file. - The returned symbol location can be used for edits or further queries. + Retrieves information on all symbols/code entities (classes, methods, etc.) based on the given name path pattern. + The returned symbol information can be used for edits or further queries. Specify `depth > 0` to retrieve children (e.g., methods of a class). - The matching behavior is determined by the structure of `name_path`, which can - either be a simple name (e.g. "method") or a name path like "class/method" (relative name path) - or "/class/method" (absolute name path). - Note that the name path is not a path in the file system but rather a path in the symbol tree - **within a single file**. Thus, file or directory names should never be included in the `name_path`. - For restricting the search to a single file or directory, pass the `relative_path` parameter. - The retrieved symbols' `name_path` attribute will always be composed of symbol names, never file - or directory names. + A name path is a path in the symbol tree *within a source file*. + For example, the method `my_method` defined in class `MyClass` would have the name path `MyClass/my_method`. + If a symbol is overloaded (e.g., in Java), a 0-based index is appended (e.g. "MyClass/my_method[0]") to + uniquely identify it. - Key aspects of the name path matching behavior: - - The name of the retrieved symbols will match the last segment of `name_path`, while preceding segments - will restrict the search to symbols that have a desired sequence of ancestors. - - If there is no `/` in `name_path`, there is no restriction on the ancestor symbols. - For example, passing `method` will match against all symbols with name paths like `method`, - `class/method`, `class/nested_class/method`, etc. - - If `name_path` contains at least one `/`, the matching is restricted to symbols - with the respective ancestors. For example, passing `class/method` will match against - `class/method` as well as `nested_class/class/method` but not `other_class/method`. - - If `name_path` starts with a `/`, it will be treated as an absolute name path pattern, i.e. - all ancestors are provided and must match. - For example, passing `/class` will match only against top-level symbols named `class` but - will not match `nested_class/class`. Passing `/class/method` will match `class/method` but - not `outer_class/class/method`. + To search for a symbol, you provide a name path pattern that is used to match against name paths. + It can be + * a simple name (e.g. "method"), which will match any symbol with that name + * a relative path like "class/method", which will match any symbol with that name path suffix + * an absolute name path "/class/method" (absolute name path), which requires an exact match of the full name path within the source file. + Append an index `[i]` to match a specific overload only, e.g. "MyClass/my_method[1]". - :param name_path: The name path pattern to search for, see above for details. - :param depth: Depth to retrieve descendants (e.g., 1 for class methods/attributes). - :param relative_path: Optional. Restrict search to this file or directory. - If None, searches entire codebase. + :param name_path_pattern: the name path matching pattern (see above) + :param depth: depth up to which descendants shall be retrieved (e.g. use 1 to also retrieve immediate children; + for the case where the symbol is a class, this will return its methods). + Default 0. + :param relative_path: Optional. Restrict search to this file or directory. If None, searches entire codebase. If a directory is passed, the search will be restricted to the files in that directory. If a file is passed, the search will be restricted to that file. If you have some knowledge about the codebase, you should use this parameter, as it will significantly @@ -62,7 +50,7 @@ class JetBrainsFindSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): """ with JetBrainsPluginClient.from_project(self.project) as client: response_dict = client.find_symbol( - name_path=name_path, + name_path=name_path_pattern, relative_path=relative_path, depth=depth, include_body=include_body, diff --git a/src/serena/tools/symbol_tools.py b/src/serena/tools/symbol_tools.py index bddb749c..f0ce46f9 100644 --- a/src/serena/tools/symbol_tools.py +++ b/src/serena/tools/symbol_tools.py @@ -85,7 +85,7 @@ class FindSymbolTool(Tool, ToolMarkerSymbolicRead): # noinspection PyDefaultArgument def apply( self, - name_path: str, + name_path_pattern: str, depth: int = 0, relative_path: str = "", include_body: bool = False, @@ -95,38 +95,26 @@ class FindSymbolTool(Tool, ToolMarkerSymbolicRead): max_answer_chars: int = -1, ) -> str: """ - Retrieves information on all symbols/code entities (classes, methods, etc.) based on the given `name_path`, - which represents a pattern for the symbol's path within the symbol tree of a single file. - The returned symbol location can be used for edits or further queries. - Specify `depth > 0` to retrieve children (e.g., methods of a class). + Retrieves information on all symbols/code entities (classes, methods, etc.) based on the given name path pattern. + The returned symbol information can be used for edits or further queries. + Specify `depth > 0` to also retrieve children/descendants (e.g., methods of a class). - The matching behavior is determined by the structure of `name_path`, which can - either be a simple name (e.g. "method") or a name path like "class/method" (relative name path) - or "/class/method" (absolute name path). Note that the name path is not a path in the file system - but rather a path in the symbol tree **within a single file**. Thus, file or directory names should never - be included in the `name_path`. For restricting the search to a single file or directory, - the `within_relative_path` parameter should be used instead. The retrieved symbols' `name_path` attribute - will always be composed of symbol names, never file or directory names. + A name path is a path in the symbol tree *within a source file*. + For example, the method `my_method` defined in class `MyClass` would have the name path `MyClass/my_method`. + If a symbol is overloaded (e.g., in Java), a 0-based index is appended (e.g. "MyClass/my_method[0]") to + uniquely identify it. - Key aspects of the name path matching behavior: - - Trailing slashes in `name_path` play no role and are ignored. - - The name of the retrieved symbols will match (either exactly or as a substring) - the last segment of `name_path`, while other segments will restrict the search to symbols that - have a desired sequence of ancestors. - - If there is no starting or intermediate slash in `name_path`, there is no - restriction on the ancestor symbols. For example, passing `method` will match - against symbols with name paths like `method`, `class/method`, `class/nested_class/method`, etc. - - If `name_path` contains a `/` but doesn't start with a `/`, the matching is restricted to symbols - with the same ancestors as the last segment of `name_path`. For example, passing `class/method` will match against - `class/method` as well as `nested_class/class/method` but not `method`. - - If `name_path` starts with a `/`, it will be treated as an absolute name path pattern, meaning - that the first segment of it must match the first segment of the symbol's name path. - For example, passing `/class` will match only against top-level symbols like `class` but not against `nested_class/class`. - Passing `/class/method` will match against `class/method` but not `nested_class/class/method` or `method`. + To search for a symbol, you provide a name path pattern that is used to match against name paths. + It can be + * a simple name (e.g. "method"), which will match any symbol with that name + * a relative path like "class/method", which will match any symbol with that name path suffix + * an absolute name path "/class/method" (absolute name path), which requires an exact match of the full name path within the source file. + Append an index `[i]` to match a specific overload only, e.g. "MyClass/my_method[1]". - - :param name_path: The name path pattern to search for, see above for details. - :param depth: Depth to retrieve descendants (e.g., 1 for class methods/attributes). + :param name_path_pattern: the name path matching pattern (see above) + :param depth: depth up to which descendants shall be retrieved (e.g. use 1 to also retrieve immediate children; + for the case where the symbol is a class, this will return its methods). + Default 0. :param relative_path: Optional. Restrict search to this file or directory. If None, searches entire codebase. If a directory is passed, the search will be restricted to the files in that directory. If a file is passed, the search will be restricted to that file. @@ -140,7 +128,8 @@ class FindSymbolTool(Tool, ToolMarkerSymbolicRead): If not provided, all kinds are included. :param exclude_kinds: Optional. List of LSP symbol kind integers to exclude. Takes precedence over `include_kinds`. If not provided, no kinds are excluded. - :param substring_matching: If True, use substring matching for the last segment of `name`. + :param substring_matching: If True, use substring matching for the last element of the pattern, such that + "Foo/get" would match "Foo/getValue" and "Foo/getData". :param max_answer_chars: Max characters for the JSON result. If exceeded, no content is returned. -1 means the default value from the config will be used. :return: a list of symbols (with locations) matching the name. @@ -149,7 +138,7 @@ class FindSymbolTool(Tool, ToolMarkerSymbolicRead): parsed_exclude_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in exclude_kinds] if exclude_kinds else None symbol_retriever = self.create_language_server_symbol_retriever() symbols = symbol_retriever.find_by_name( - name_path, + name_path_pattern, include_body=include_body, include_kinds=parsed_include_kinds, exclude_kinds=parsed_exclude_kinds, diff --git a/test/serena/test_serena_agent.py b/test/serena/test_serena_agent.py index 882b45be..e2ac4c62 100644 --- a/test/serena/test_serena_agent.py +++ b/test/serena/test_serena_agent.py @@ -86,7 +86,7 @@ class TestSerenaAgent: def test_find_symbol(self, serena_agent, symbol_name: str, expected_kind: str, expected_file: str): agent = serena_agent find_symbol_tool = agent.get_tool(FindSymbolTool) - result = find_symbol_tool.apply_ex(name_path=symbol_name) + result = find_symbol_tool.apply_ex(name_path_pattern=symbol_name) symbols = json.loads(result) assert any( @@ -138,7 +138,7 @@ class TestSerenaAgent: # Find the symbol location first find_symbol_tool = agent.get_tool(FindSymbolTool) - result = find_symbol_tool.apply_ex(name_path=symbol_name, relative_path=def_file) + result = find_symbol_tool.apply_ex(name_path_pattern=symbol_name, relative_path=def_file) time.sleep(1) symbols = json.loads(result) @@ -233,7 +233,7 @@ class TestSerenaAgent: find_symbol_tool = agent.get_tool(FindSymbolTool) result = find_symbol_tool.apply_ex( - name_path=name_path, + name_path_pattern=name_path, depth=0, relative_path=None, include_body=False, @@ -277,7 +277,7 @@ class TestSerenaAgent: find_symbol_tool = agent.get_tool(FindSymbolTool) result = find_symbol_tool.apply_ex( - name_path=name_path, + name_path_pattern=name_path, depth=0, substring_matching=True, ) From 83f4da5ff0972464fd267d0e8ca4a48af43e90d2 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 11 Nov 2025 14:45:12 +0100 Subject: [PATCH 11/26] Add profiling script, add pyinstrument to dev dependencies --- pyproject.toml | 3 ++- scripts/profile_tool_call.py | 52 ++++++++++++++++++++++++++++++++++++ uv.lock | 20 +++++++++++++- 3 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 scripts/profile_tool_call.py diff --git a/pyproject.toml b/pyproject.toml index 34348aac..86f29a47 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,8 @@ dev = [ "sphinx_rtd_theme>=0.5.1", "sphinx-toolbox==3.7.0", "jupyter-book>=1,<2", - "nbsphinx" + "nbsphinx", + "pyinstrument", ] agno = ["agno>=2.2.1", "sqlalchemy>=2.0.40"] google = ["google-genai>=1.8.0"] diff --git a/scripts/profile_tool_call.py b/scripts/profile_tool_call.py new file mode 100644 index 00000000..8fe922a5 --- /dev/null +++ b/scripts/profile_tool_call.py @@ -0,0 +1,52 @@ +import cProfile +from pathlib import Path +from typing import Literal + +from sensai.util import logging +from sensai.util.logging import LogTime +from sensai.util.profiling import profiled + +from serena.agent import SerenaAgent +from serena.config.serena_config import SerenaConfig +from serena.tools import FindSymbolTool + +log = logging.getLogger(__name__) + + +if __name__ == "__main__": + logging.configure() + + # The profiler to use: + # Use pyinstrument for hierarchical profiling output + # Use cProfile to determine which functions take the most time overall (and use snakeviz to visualize) + profiler: Literal["pyinstrument", "cprofile"] = "cprofile" + + project_path = Path(__file__).parent.parent # Serena root + + serena_config = SerenaConfig.from_config_file() + serena_config.log_level = logging.INFO + serena_config.gui_log_window_enabled = False + serena_config.web_dashboard = False + + agent = SerenaAgent(str(project_path), serena_config=serena_config) + + # wait for language server to be ready + agent.execute_task(lambda: log.info("Language server is ready.")) + + def tool_call(): + """This is the function we want to profile.""" + # NOTE: We use apply (not apply_ex) to run the tool call directly on the main thread + with LogTime("Tool call"): + result = agent.get_tool(FindSymbolTool).apply(name_path="DQN") + log.info("Tool result:\n%s", result) + + if profiler == "pyinstrument": + + @profiled(log_to_file=True) + def profiled_tool_call(): + tool_call() + + profiled_tool_call() + + elif profiler == "cprofile": + cProfile.run("tool_call()", "tool_call.pstat") diff --git a/uv.lock b/uv.lock index 7e0d86d1..538dfd92 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = "==3.11.*" [[package]] @@ -1524,6 +1524,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, ] +[[package]] +name = "pyinstrument" +version = "5.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/80/ce/824ee634994e612156f7b84eaf50b8523c676ebfed8d8dd12939a82f4c15/pyinstrument-5.1.1.tar.gz", hash = "sha256:bc401cda990b3c1cfe8e0e0473cbd605df3c63b73478a89ac4ab108f2184baa8", size = 264730, upload-time = "2025-08-12T11:35:43.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/65/dc0fcc6122e6f484ffa48260d1023e1fe90e53da8d69a7de656205af91ba/pyinstrument-5.1.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5272cff6eea21163b2105f6a80c907315e0f567720621e6d5672dc01bf71ee48", size = 130287, upload-time = "2025-08-12T11:34:32.436Z" }, + { url = "https://files.pythonhosted.org/packages/90/96/8a6cab312342f1ad7322b459dcbdb2e041251b7bbe4c1d0dd38ccbf2ae20/pyinstrument-5.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d4e7dc5a4aee37a44ff2e63db3127f2044dd95edcae240cb95915adbf223d4be", size = 122879, upload-time = "2025-08-12T11:34:33.506Z" }, + { url = "https://files.pythonhosted.org/packages/8a/89/1ad1ae703951832a34ec6b32354c6df1d0690b0333b7b8396c92afcdc23e/pyinstrument-5.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:602d55121df1d88aeb6d8ebc801597fdcb9718f78d602ae81458d65c56f25d24", size = 146412, upload-time = "2025-08-12T11:34:34.975Z" }, + { url = "https://files.pythonhosted.org/packages/a5/70/588ec07eeaa5d5d4aff2d47c8010379e9b39a7aee7e3fda625a0ce4f5838/pyinstrument-5.1.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63b5a788ff955e0597bc95463e64d5fa3747017524fdc02a0f5d12d5117cf2b9", size = 144885, upload-time = "2025-08-12T11:34:36.546Z" }, + { url = "https://files.pythonhosted.org/packages/0a/8f/119ad44d454edb927434bb53d8a69904ecaa47ec56f95c688c557a9590de/pyinstrument-5.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7f66038ec55a12e5510689240cdc745f8e98c90b93363f745106976e5cfb7397", size = 145530, upload-time = "2025-08-12T11:34:37.726Z" }, + { url = "https://files.pythonhosted.org/packages/8e/31/d53b20d1967ba78574807c5940251c0f238e8e3515ee9c4f20207eb09199/pyinstrument-5.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:110a08b6e7fa9542eb37c337e79467913d364a03bc2062f85566ba96bc82f54e", size = 144708, upload-time = "2025-08-12T11:34:38.855Z" }, + { url = "https://files.pythonhosted.org/packages/48/18/78ccb35dd265e83c0b7404a501004387b2e2390a0a44b6e4a004307931d1/pyinstrument-5.1.1-cp311-cp311-win32.whl", hash = "sha256:a223d5e9226ccede5bf2fbd4d13ce0aeb5120501b633ba85290ed94df37d3623", size = 124151, upload-time = "2025-08-12T11:34:40.392Z" }, + { url = "https://files.pythonhosted.org/packages/43/c0/3eb94aa2c2f0b8d49e290d4df662e21e707eb4a23f3b938239634fbfdc17/pyinstrument-5.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:16ce582a7b56287d338a8b59688458341aab5c6abda970ba50b2f7b3fd69f89d", size = 124942, upload-time = "2025-08-12T11:34:41.564Z" }, +] + [[package]] name = "pyright" version = "1.1.403" @@ -1875,6 +1891,7 @@ dev = [ { name = "mypy" }, { name = "nbsphinx" }, { name = "poethepoet" }, + { name = "pyinstrument" }, { name = "pytest" }, { name = "pytest-xdist" }, { name = "ruff" }, @@ -1912,6 +1929,7 @@ requires-dist = [ { name = "poethepoet", marker = "extra == 'dev'", specifier = ">=0.20.0" }, { name = "psutil", specifier = ">=7.0.0" }, { name = "pydantic", specifier = ">=2.10.6" }, + { name = "pyinstrument", marker = "extra == 'dev'" }, { name = "pyright", specifier = ">=1.1.396,<2" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.2" }, { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.5.0" }, From 24ea449abfafe626bcb56434819ab6306ef81f67 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 11 Nov 2025 17:36:04 +0100 Subject: [PATCH 12/26] Introduce "raw" document symbol cache, removing the high-level cache for the time being (i.e. cache the raw LS result) Reuse the legacy cache v23-06-25 as the raw symbol cache (the stored root symbols are just augmented versions of the originally returned LS response) --- .../language_servers/eclipse_jdtls.py | 4 +- src/solidlsp/ls.py | 209 +++++++++++------- 2 files changed, 136 insertions(+), 77 deletions(-) diff --git a/src/solidlsp/language_servers/eclipse_jdtls.py b/src/solidlsp/language_servers/eclipse_jdtls.py index 8ade5b70..c87f97f4 100644 --- a/src/solidlsp/language_servers/eclipse_jdtls.py +++ b/src/solidlsp/language_servers/eclipse_jdtls.py @@ -789,8 +789,8 @@ class EclipseJDTLS(SolidLanguageServer): # TODO: Add comments about why we wait here, and how this can be optimized self.service_ready_event.wait() - def _request_document_symbols(self, relative_file_path: str, include_body: bool = False) -> list[GenericDocumentSymbol]: - result = super()._request_document_symbols(relative_file_path, include_body) + def _request_document_symbols(self, relative_file_path: str) -> list[GenericDocumentSymbol]: + result = super()._request_document_symbols(relative_file_path) # JDTLS sometimes returns symbol names with type information to handle overloads, # e.g. "myMethod(int) ", but we want overloads to be handled via overload_idx, diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index 5092cf1f..0d2f31ff 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -17,6 +17,7 @@ from time import sleep from typing import Self, Union, cast import pathspec +from sensai.util.pickle import load_pickle from serena.text_utils import MatchedConsecutiveLines from serena.util.file_system import match_path @@ -40,6 +41,7 @@ from solidlsp.settings import SolidLSPSettings from solidlsp.util.cache import load_cache, save_cache GenericDocumentSymbol = Union[LSPTypes.DocumentSymbol, LSPTypes.SymbolInformation, ls_types.UnifiedSymbolInformation] +log = logging.getLogger(__name__) @dataclasses.dataclass(kw_only=True) @@ -85,6 +87,9 @@ class SolidLanguageServer(ABC): """ CACHE_FOLDER_NAME = "cache" + CACHE_VERSION_RAW_DOCUMENT_SYMBOLS = 1 + CACHE_FILE_RAW_DOCUMENT_SYMBOLS = "raw_document_symbols.pkl" + CACHE_FILE_RAW_DOCUMENT_SYMBOLS_LEGACY = "document_symbols_cache_v23-06-25.pkl" CACHE_VERSION = 2 # To be overridden and extended by subclasses @@ -162,6 +167,7 @@ class SolidLanguageServer(ABC): process_launch_info: ProcessLaunchInfo, language_id: str, solidlsp_settings: SolidLSPSettings, + cache_version_raw_document_symbols: int = 1, ): """ Initializes a LanguageServer instance. @@ -175,6 +181,10 @@ class SolidLanguageServer(ABC): This parameter is the command to launch the language server process. The command must pass appropriate flags to the binary, so that it runs in the stdio mode, as opposed to HTTP, TCP modes supported by some language servers. + :param cache_version_raw_document_symbols: the version, for caching, of the raw document symbols coming + from this specific language server. This should be incremented by subclasses calling this constructor + whenever the format of the raw document symbols changes (typically because the language server + improves/fixes its output). """ self._solidlsp_settings = solidlsp_settings self._encoding = config.encoding @@ -189,14 +199,16 @@ class SolidLanguageServer(ABC): self.open_file_buffers: dict[str, LSPFileBuffer] = {} self.language = Language(language_id) - # load cache first to prevent any racing conditions due to asyncio stuff - self._document_symbols_cache: dict[ - str, tuple[str, tuple[list[ls_types.UnifiedSymbolInformation], list[ls_types.UnifiedSymbolInformation]]] - ] = {} - """Maps file paths to a tuple of (file_content_hash, result_of_request_document_symbols)""" - self._cache_lock = threading.Lock() - self._cache_has_changed: bool = False - self.load_cache() + # load caches + self._cache_dir = ( + Path(self.repository_root_path) / self._solidlsp_settings.project_data_relative_path / self.CACHE_FOLDER_NAME / self.language_id + ) + self._cache_dir.mkdir(parents=True, exist_ok=True) + self._raw_document_symbols_cache_version = cache_version_raw_document_symbols + self._raw_document_symbols_cache: dict[str, tuple[str, list[GenericDocumentSymbol]]] = {} + """Maps relative file paths to a tuple of (file_content_hash, root_symbols)""" + self._raw_document_symbols_cache_is_modified: bool = False + self._load_raw_document_symbols_cache() self.server_started = False self.completions_available = threading.Event() @@ -831,22 +843,53 @@ class SolidLanguageServer(ABC): return [json.loads(json_repr) for json_repr in set(json.dumps(item, sort_keys=True) for item in completions_list)] - def _request_document_symbols(self, relative_file_path: str, include_body: bool = False) -> list[GenericDocumentSymbol]: + def _request_document_symbols(self, relative_file_path: str, file_data: LSPFileBuffer | None) -> list[GenericDocumentSymbol]: """ Sends a [documentSymbol](https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_documentSymbol) - request to the language server to find symbols in the given file, which is assumed to be opened while the call is active. - Waits for the response and returns it. + request to the language server to find symbols in the given file - or returns a cached result if available. :param relative_file_path: the relative path of the file that has the symbols. - IMPORTANT: It must be ensured that the file is already opened in the Language Server before calling this method. - :param include_body: whether to include the body of the symbols in the result. - :return: the list of symbols in the file. + :param file_data: the file data buffer, if already opened. If None, the file will be opened in this method. + :return: the list of root symbols in the file. """ - self.logger.log(f"Requesting document symbols for {relative_file_path} from the Language Server", logging.DEBUG) - response = self.server.send.document_symbol( - {"textDocument": {"uri": pathlib.Path(os.path.join(self.repository_root_path, relative_file_path)).as_uri()}} - ) - return response + + def get_cached_result(cache_key: str, fd: LSPFileBuffer) -> list[GenericDocumentSymbol] | None: + file_hash_and_result = self._raw_document_symbols_cache.get(cache_key) + if file_hash_and_result is not None: + file_hash, result = file_hash_and_result + if file_hash == fd.content_hash: + log.debug("Returning cached document symbols for %s", relative_file_path) + return result + else: + log.debug("Content for %s has changed. Will overwrite in-memory cache", relative_file_path) + else: + log.debug("No cache hit for symbols in %s", relative_file_path) + return None + + def get_symbols(fd: LSPFileBuffer) -> list[GenericDocumentSymbol]: + # check for cached result + cache_key = relative_file_path + response = get_cached_result(cache_key, fd) + if response is not None: + return response + + # no cached result, query language server + self.logger.log(f"Requesting document symbols for {relative_file_path} from the Language Server", logging.DEBUG) + response = self.server.send.document_symbol( + {"textDocument": {"uri": pathlib.Path(os.path.join(self.repository_root_path, relative_file_path)).as_uri()}} + ) + + # update cache + self._raw_document_symbols_cache[cache_key] = (file_data.content_hash, response) + self._raw_document_symbols_cache_is_modified = True + + return response + + if file_data is not None: + return get_symbols(file_data) + else: + with self.open_file(relative_file_path) as opened_file_data: + return get_symbols(opened_file_data) def request_document_symbols( self, relative_file_path: str, include_body: bool = False @@ -864,12 +907,11 @@ class SolidLanguageServer(ABC): If you need a symbol tree that contains file symbols as well, you should use `request_full_symbol_tree` instead. """ # check if the desired result is cached - # TODO: it's kinda dumb to not use the cache if include_body is False after include_body was True once - # Should be fixed in the future, it's a small performance optimization - cache_key = f"{relative_file_path}-{include_body}" with self.open_file(relative_file_path) as file_data: + """ + cache_key = f"{relative_file_path}-{include_body}" with self._cache_lock: - file_hash_and_result = self._document_symbols_cache.get(cache_key) + file_hash_and_result = self._raw_document_symbols_cache.get(cache_key) if file_hash_and_result is not None: file_hash, result = file_hash_and_result if file_hash == file_data.content_hash: @@ -879,9 +921,10 @@ class SolidLanguageServer(ABC): self.logger.log(f"Content for {relative_file_path} has changed. Will overwrite in-memory cache", logging.DEBUG) else: self.logger.log(f"No cache hit for symbols with {include_body=} in {relative_file_path}", logging.DEBUG) + """ # request the symbols from the language server - response = self._request_document_symbols(relative_file_path, include_body=include_body) + response = self._request_document_symbols(relative_file_path, file_data) if response is None: self.logger.log( @@ -897,7 +940,7 @@ class SolidLanguageServer(ABC): logging.DEBUG, ) - def turn_item_into_symbol_with_children(item: GenericDocumentSymbol) -> ls_types.UnifiedSymbolInformation: + def turn_item_into_unified_symbol(item: GenericDocumentSymbol) -> ls_types.UnifiedSymbolInformation: """ Ensures that the given item is augmented with location, children, and body (if requested), such that it satisfies the UnifiedSymbolInformation structure. @@ -924,7 +967,7 @@ class SolidLanguageServer(ABC): location["absolutePath"] = absolute_path if "relativePath" not in location: location["relativePath"] = relative_file_path - if include_body: + if "body" not in item: item["body"] = self.retrieve_symbol_body(item) # handle missing selectionRange if "selectionRange" not in item: @@ -951,7 +994,7 @@ class SolidLanguageServer(ABC): name_counts = defaultdict(lambda: 0) for node in nodes: flat_all_symbol_list.append(node) - usymbol_node = turn_item_into_symbol_with_children(node) + usymbol_node = turn_item_into_unified_symbol(node) if total_name_counts[usymbol_node["name"]] > 1: usymbol_node["overload_idx"] = name_counts[usymbol_node["name"]] name_counts[usymbol_node["name"]] += 1 @@ -965,10 +1008,12 @@ class SolidLanguageServer(ABC): result = flat_all_symbol_list, root_nodes # update cache + """ self.logger.log(f"Caching document symbols for {relative_file_path}", logging.DEBUG) with self._cache_lock: - self._document_symbols_cache[cache_key] = (file_data.content_hash, result) - self._cache_has_changed = True + self._raw_document_symbols_cache[cache_key] = (file_data.content_hash, result) + self._raw_document_symbols_cache_has_changed = True + """ return result @@ -1566,55 +1611,69 @@ class SolidLanguageServer(ABC): return defining_symbol - @property - def cache_path(self) -> Path: - """ - The path to the cache file for the document symbols. - """ - return ( - Path(self.repository_root_path) - / self._solidlsp_settings.project_data_relative_path - / self.CACHE_FOLDER_NAME - / self.language_id - / "document_symbols_cache.pkl" - ) + def _save_raw_document_symbols_cache(self): + cache_file = self._cache_dir / self.CACHE_FILE_RAW_DOCUMENT_SYMBOLS + cache_version = (self.CACHE_VERSION_RAW_DOCUMENT_SYMBOLS, self._raw_document_symbols_cache_version) - def save_cache(self): - with self._cache_lock: - if not self._cache_has_changed: - self.logger.log("No changes to document symbols cache, skipping save", logging.DEBUG) - return - - self.logger.log(f"Saving updated document symbols cache to {self.cache_path}", logging.INFO) - self.cache_path.parent.mkdir(parents=True, exist_ok=True) - try: - save_cache(str(self.cache_path), self.CACHE_VERSION, self._document_symbols_cache) - self._cache_has_changed = False - except Exception as e: - self.logger.log( - f"Failed to save document symbols cache to {self.cache_path}: {e}. " - "Note: this may have resulted in a corrupted cache file.", - logging.ERROR, - ) - - def load_cache(self): - if not self.cache_path.exists(): + if not self._raw_document_symbols_cache_is_modified: + log.debug("No changes to raw document symbols cache, skipping save") return - with self._cache_lock: - self.logger.log(f"Loading document symbols cache from {self.cache_path}", logging.INFO) - try: - saved_cache = load_cache(str(self.cache_path), self.CACHE_VERSION) - if saved_cache is not None: - self._document_symbols_cache = saved_cache - self.logger.log(f"Loaded {len(self._document_symbols_cache)} document symbols from cache.", logging.INFO) - except Exception as e: - # cache often becomes corrupt, so just skip loading it - self.logger.log( - f"Failed to load document symbols cache from {self.cache_path}: {e}. Possible cause: the cache file is corrupted. " - "Check for any errors related to saving the cache in the logs.", - logging.ERROR, - ) + log.info("Saving updated raw document symbols cache to %s", cache_file) + try: + save_cache(str(cache_file), cache_version, self._raw_document_symbols_cache) + self._raw_document_symbols_cache_is_modified = False + except Exception as e: + log.error( + "Failed to save raw document symbols cache to %s: %s. Note: this may have resulted in a corrupted cache file.", + cache_file, + e, + ) + + def save_cache(self): + self._save_raw_document_symbols_cache() + + def _load_raw_document_symbols_cache(self): + cache_file = self._cache_dir / self.CACHE_FILE_RAW_DOCUMENT_SYMBOLS + cache_version = (self.CACHE_VERSION_RAW_DOCUMENT_SYMBOLS, self._raw_document_symbols_cache_version) + + if not cache_file.exists(): + # check for legacy cache to load to migrate + legacy_cache_file = self._cache_dir / self.CACHE_FILE_RAW_DOCUMENT_SYMBOLS_LEGACY + if legacy_cache_file.exists(): + try: + legacy_cache: dict[ + str, tuple[str, tuple[list[ls_types.UnifiedSymbolInformation], list[ls_types.UnifiedSymbolInformation]]] + ] = load_pickle(legacy_cache_file) + log.info("Migrating legacy document symbols cache with %d entries", len(legacy_cache)) + num_symbols_migrated = 0 + migrated_cache = {} + for cache_key, (file_hash, (all_symbols, root_symbols)) in legacy_cache.items(): + if cache_key.endswith("-True"): # include_body=True + new_cache_key = cache_key[:-5] + migrated_cache[new_cache_key] = (file_hash, root_symbols) + num_symbols_migrated += len(all_symbols) + log.info("Migrated %d document symbols from legacy cache", num_symbols_migrated) + self._raw_document_symbols_cache = migrated_cache + return + except Exception as e: + log.error("Failed to migrate cache: %s", e) + return + + # cache file exists, so load it + log.info(f"Loading document symbols cache from %s", cache_file) + try: + saved_cache = load_cache(str(cache_file), cache_version) + if saved_cache is not None: + self._raw_document_symbols_cache = saved_cache + self.logger.log(f"Loaded {len(self._raw_document_symbols_cache)} entries from document symbols cache.", logging.INFO) + except Exception as e: + # cache can become corrupt, so just skip loading it + log.warning( + "Failed to load document symbols cache from %s (%s); Ignoring cache.", + cache_file, + e, + ) def request_workspace_symbol(self, query: str) -> list[ls_types.UnifiedSymbolInformation] | None: """ From d8cac3aedb49cdbe1cf83475a7d47defda493485 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 11 Nov 2025 20:15:19 +0100 Subject: [PATCH 13/26] Introduce abstraction DocumentSymbols, which is now returned by request_document_symbols --- src/serena/cli.py | 3 +- src/serena/symbol.py | 10 +-- .../language_servers/al_language_server.py | 4 +- .../language_servers/bash_language_server.py | 15 ++-- .../fortran_language_server.py | 13 ++-- src/solidlsp/language_servers/nixd_ls.py | 13 ++-- src/solidlsp/ls.py | 75 ++++++++++++++----- test/solidlsp/al/test_al_basic.py | 6 +- test/solidlsp/bash/test_bash_basic.py | 16 ++-- test/solidlsp/clojure/test_clojure_basic.py | 2 +- test/solidlsp/csharp/test_csharp_basic.py | 8 +- test/solidlsp/dart/test_dart_basic.py | 8 +- test/solidlsp/elixir/test_elixir_basic.py | 10 +-- .../elixir/test_elixir_ignored_dirs.py | 4 +- .../elixir/test_elixir_integration.py | 2 +- .../elixir/test_elixir_symbol_retrieval.py | 2 +- test/solidlsp/elm/test_elm_basic.py | 4 +- test/solidlsp/erlang/test_erlang_basic.py | 2 +- .../erlang/test_erlang_ignored_dirs.py | 6 +- .../erlang/test_erlang_symbol_retrieval.py | 6 +- test/solidlsp/fortran/test_fortran_basic.py | 10 +-- test/solidlsp/go/test_go_basic.py | 2 +- test/solidlsp/haskell/test_haskell_basic.py | 6 +- test/solidlsp/java/test_java_basic.py | 2 +- test/solidlsp/julia/test_julia_basic.py | 2 +- test/solidlsp/kotlin/test_kotlin_basic.py | 2 +- test/solidlsp/lua/test_lua_basic.py | 12 +-- test/solidlsp/markdown/test_markdown_basic.py | 8 +- test/solidlsp/nix/test_nix_basic.py | 14 ++-- test/solidlsp/perl/test_perl_basic.py | 2 +- test/solidlsp/python/test_python_basic.py | 8 +- test/solidlsp/python/test_symbol_retrieval.py | 14 ++-- test/solidlsp/r/test_r_basic.py | 2 +- test/solidlsp/rego/test_rego_basic.py | 10 +-- test/solidlsp/ruby/test_ruby_basic.py | 2 +- .../ruby/test_ruby_symbol_retrieval.py | 12 +-- test/solidlsp/rust/test_rust_2024_edition.py | 6 +- test/solidlsp/rust/test_rust_basic.py | 4 +- .../scala/test_scala_language_server.py | 2 +- test/solidlsp/swift/test_swift_basic.py | 8 +- .../terraform/test_terraform_basic.py | 6 +- .../typescript/test_typescript_basic.py | 2 +- test/solidlsp/zig/test_zig_basic.py | 18 ++--- 43 files changed, 198 insertions(+), 165 deletions(-) diff --git a/src/serena/cli.py b/src/serena/cli.py index fb78cd3d..3d73b4c9 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -538,7 +538,8 @@ class ProjectCommands(AutoRegisteringGroup): try: for ls in ls_mgr.iter_language_servers(): click.echo(f"Indexing for language {ls.language.value} …") - symbols, _ = ls.request_document_symbols(file, include_body=False) + document_symbols = ls.request_document_symbols(file, include_body=False) + symbols, _ = document_symbols.get_all_symbols_and_roots() ls.request_document_symbols(file, include_body=True) if verbose: click.echo(f"Symbols in file '{file}':") diff --git a/src/serena/symbol.py b/src/serena/symbol.py index 589c0d20..f35294c7 100644 --- a/src/serena/symbol.py +++ b/src/serena/symbol.py @@ -513,18 +513,12 @@ class LanguageServerSymbolRetriever: ) return symbols - def get_document_symbols(self, relative_path: str) -> list[LanguageServerSymbol]: - lang_server = self.get_language_server(relative_path) - symbol_dicts, _roots = lang_server.request_document_symbols(relative_path, include_body=False) - symbols = [LanguageServerSymbol(s) for s in symbol_dicts] - return symbols - def find_by_location(self, location: LanguageServerSymbolLocation) -> LanguageServerSymbol | None: if location.relative_path is None: return None lang_server = self.get_language_server(location.relative_path) - symbol_dicts, _roots = lang_server.request_document_symbols(location.relative_path, include_body=False) - for symbol_dict in symbol_dicts: + document_symbols = lang_server.request_document_symbols(location.relative_path, include_body=False) + for symbol_dict in document_symbols.iter_symbols(): symbol = LanguageServerSymbol(symbol_dict) if symbol.location == location: return symbol diff --git a/src/solidlsp/language_servers/al_language_server.py b/src/solidlsp/language_servers/al_language_server.py index 3745b223..e2786f10 100644 --- a/src/solidlsp/language_servers/al_language_server.py +++ b/src/solidlsp/language_servers/al_language_server.py @@ -713,7 +713,7 @@ class ALLanguageServer(SolidLanguageServer): if os.path.isfile(within_abs_path): # Single file case - use parent class implementation - _, root_nodes = self.request_document_symbols(within_relative_path, include_body=include_body) + root_nodes = self.request_document_symbols(within_relative_path, include_body=include_body).root_symbols return root_nodes # Directory case - scan within this directory @@ -755,7 +755,7 @@ class ALLanguageServer(SolidLanguageServer): try: # Use our overridden request_document_symbols which handles opening self.logger.log(f"AL: Getting symbols for {relative_path}", logging.DEBUG) - all_syms, root_syms = self.request_document_symbols(relative_path, include_body=include_body) + all_syms, root_syms = self.request_document_symbols(relative_path, include_body=include_body).get_all_symbols_and_roots() if root_syms: # Create a file-level symbol containing the document symbols diff --git a/src/solidlsp/language_servers/bash_language_server.py b/src/solidlsp/language_servers/bash_language_server.py index d8f780e2..60957425 100644 --- a/src/solidlsp/language_servers/bash_language_server.py +++ b/src/solidlsp/language_servers/bash_language_server.py @@ -9,9 +9,8 @@ import pathlib import shutil import threading -from solidlsp import ls_types from solidlsp.language_servers.common import RuntimeDependency, RuntimeDependencyCollection -from solidlsp.ls import SolidLanguageServer +from solidlsp.ls import DocumentSymbols, SolidLanguageServer from solidlsp.ls_config import LanguageServerConfig from solidlsp.ls_logger import LanguageServerLogger from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams @@ -194,9 +193,7 @@ class BashLanguageServer(SolidLanguageServer): else: self.logger.log("Bash server initialization complete", logging.INFO) - def request_document_symbols( - self, relative_file_path: str, include_body: bool = False - ) -> tuple[list[ls_types.UnifiedSymbolInformation], list[ls_types.UnifiedSymbolInformation]]: + def request_document_symbols(self, relative_file_path: str, include_body: bool = False) -> DocumentSymbols: """ Request document symbols from bash-language-server via LSP. @@ -212,19 +209,19 @@ class BashLanguageServer(SolidLanguageServer): include_body: Whether to include function bodies in symbol information Returns: - Tuple of (all_symbols, root_symbols) detected by the LSP server + The symbols """ self.logger.log(f"Requesting document symbols via LSP for {relative_file_path}", logging.DEBUG) # Use the standard LSP approach - bash-language-server handles all function syntaxes correctly - all_symbols, root_symbols = super().request_document_symbols(relative_file_path, include_body) + document_symbols = super().request_document_symbols(relative_file_path, include_body) # Log detection results for debugging - functions = [s for s in all_symbols if s.get("kind") == 12] + functions = [s for s in document_symbols.iter_symbols() if s.get("kind") == 12] self.logger.log( f"LSP function detection for {relative_file_path}: Found {len(functions)} functions", logging.INFO, ) - return all_symbols, root_symbols + return document_symbols diff --git a/src/solidlsp/language_servers/fortran_language_server.py b/src/solidlsp/language_servers/fortran_language_server.py index a77135a6..6858d41d 100644 --- a/src/solidlsp/language_servers/fortran_language_server.py +++ b/src/solidlsp/language_servers/fortran_language_server.py @@ -12,7 +12,7 @@ import threading from overrides import override from solidlsp import ls_types -from solidlsp.ls import SolidLanguageServer +from solidlsp.ls import DocumentSymbols, SolidLanguageServer from solidlsp.ls_config import LanguageServerConfig from solidlsp.ls_logger import LanguageServerLogger from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams @@ -138,9 +138,7 @@ class FortranLanguageServer(SolidLanguageServer): return symbol @override - def request_document_symbols( - self, relative_file_path: str, include_body: bool = False - ) -> tuple[list[ls_types.UnifiedSymbolInformation], list[ls_types.UnifiedSymbolInformation]]: + def request_document_symbols(self, relative_file_path: str, include_body: bool = False) -> DocumentSymbols: """ Override to fix fortls's incorrect selectionRange bug. @@ -155,7 +153,7 @@ class FortranLanguageServer(SolidLanguageServer): """ # Get symbols from fortls (with incorrect selectionRange) - all_symbols, root_symbols = super().request_document_symbols(relative_file_path, include_body) + document_symbols = super().request_document_symbols(relative_file_path, include_body) # Get file content for parsing with self.open_file(relative_file_path) as file_data: @@ -173,10 +171,9 @@ class FortranLanguageServer(SolidLanguageServer): return fixed # Apply fix to all symbols - fixed_all_symbols = [fix_symbol_and_children(sym) for sym in all_symbols] - fixed_root_symbols = [fix_symbol_and_children(sym) for sym in root_symbols] + fixed_root_symbols = [fix_symbol_and_children(sym) for sym in document_symbols.root_symbols] - return fixed_all_symbols, fixed_root_symbols + return DocumentSymbols(fixed_root_symbols) @staticmethod def _check_fortls_installation(): diff --git a/src/solidlsp/language_servers/nixd_ls.py b/src/solidlsp/language_servers/nixd_ls.py index bf108967..08927052 100644 --- a/src/solidlsp/language_servers/nixd_ls.py +++ b/src/solidlsp/language_servers/nixd_ls.py @@ -16,7 +16,7 @@ from pathlib import Path from overrides import override from solidlsp import ls_types -from solidlsp.ls import SolidLanguageServer +from solidlsp.ls import DocumentSymbols, SolidLanguageServer from solidlsp.ls_config import LanguageServerConfig from solidlsp.ls_logger import LanguageServerLogger from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams @@ -70,9 +70,7 @@ class NixLanguageServer(SolidLanguageServer): return symbol @override - def request_document_symbols( - self, relative_file_path: str, include_body: bool = False - ) -> tuple[list[ls_types.UnifiedSymbolInformation], list[ls_types.UnifiedSymbolInformation]]: + def request_document_symbols(self, relative_file_path: str, include_body: bool = False) -> DocumentSymbols: """ Override to extend Nix symbol ranges to include trailing semicolons. @@ -80,7 +78,7 @@ class NixLanguageServer(SolidLanguageServer): statement-level ranges (including semicolons) for proper symbol replacement. """ # Get symbols from parent implementation - all_symbols, root_symbols = super().request_document_symbols(relative_file_path, include_body) + document_symbols = super().request_document_symbols(relative_file_path, include_body) # Get file content for range extension file_content = self.language_server.retrieve_full_file_content(relative_file_path) @@ -97,10 +95,9 @@ class NixLanguageServer(SolidLanguageServer): return extended # Apply range extension to all symbols - extended_all_symbols = [extend_symbol_and_children(sym) for sym in all_symbols] - extended_root_symbols = [extend_symbol_and_children(sym) for sym in root_symbols] + extended_root_symbols = [extend_symbol_and_children(sym) for sym in document_symbols.root_symbols] - return extended_all_symbols, extended_root_symbols + return DocumentSymbols(extended_root_symbols) @override def is_ignored_dirname(self, dirname: str) -> bool: diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index 0d2f31ff..641b355a 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -17,7 +17,7 @@ from time import sleep from typing import Self, Union, cast import pathspec -from sensai.util.pickle import load_pickle +from sensai.util.pickle import getstate, load_pickle from serena.text_utils import MatchedConsecutiveLines from serena.util.file_system import match_path @@ -80,6 +80,46 @@ class LSPFileBuffer: self.content_hash = hashlib.md5(self.contents.encode("utf-8")).hexdigest() +class DocumentSymbols: + # IMPORTANT: Instances of this class are persisted in the high-level document symbol cache + + def __init__(self, root_symbols: list[ls_types.UnifiedSymbolInformation]): + self.root_symbols = root_symbols + self._all_symbols: list[ls_types.UnifiedSymbolInformation] | None = None + + def __getstate__(self): + return getstate(DocumentSymbols, self, transient_properties=["_all_symbols"]) + + def iter_symbols(self) -> Iterator[ls_types.UnifiedSymbolInformation]: + """ + Iterate over all symbols in the document symbol tree. + Yields symbols in a depth-first manner. + """ + if self._all_symbols is not None: + yield from self._all_symbols + return + + def traverse(s: ls_types.UnifiedSymbolInformation): + yield s + for child in s.get("children", []): + yield from traverse(child) + + for root_symbol in self.root_symbols: + yield from traverse(root_symbol) + + def get_all_symbols_and_roots(self) -> tuple[list[ls_types.UnifiedSymbolInformation], list[ls_types.UnifiedSymbolInformation]]: + """ + This function returns all symbols in the document as a flat list and the root symbols. + It exists to facilitate migration from previous versions, where this was the return interface of + the LS method that obtained document symbols. + + :return: A tuple containing a list of all symbols in the document and a list of root symbols. + """ + if self._all_symbols is None: + self._all_symbols = list(self.iter_symbols()) + return self._all_symbols, self.root_symbols + + class SolidLanguageServer(ABC): """ The LanguageServer class provides a language agnostic interface to the Language Server Protocol. @@ -891,16 +931,14 @@ class SolidLanguageServer(ABC): with self.open_file(relative_file_path) as opened_file_data: return get_symbols(opened_file_data) - def request_document_symbols( - self, relative_file_path: str, include_body: bool = False - ) -> tuple[list[ls_types.UnifiedSymbolInformation], list[ls_types.UnifiedSymbolInformation]]: + def request_document_symbols(self, relative_file_path: str, include_body: bool = False) -> DocumentSymbols: """ - Retrieves the full list of symbols in the given file, along with the root symbols that represent the tree structure of the symbols. + Retrieves the collection of symbols in the given file :param relative_file_path: The relative path of the file that has the symbols :param include_body: whether to include the body of the symbols in the result. - :return: A list of symbols in the file, and a list of root symbols that represent the tree structure of the symbols. - All symbols will have a location, children, and a parent attribute, + :return: the collection of symbols in the file. + All contained symbols will have a location, children, and a parent attribute, where the parent attribute is None for root symbols. Note that this is slightly different from the call to request_full_symbol_tree, where the parent attribute will be the file symbol which in turn may have a package symbol as parent. @@ -1004,8 +1042,7 @@ class SolidLanguageServer(ABC): root_nodes: list[ls_types.UnifiedSymbolInformation] = response convert_nodes_with_common_parent(root_nodes, None) - - result = flat_all_symbol_list, root_nodes + result = DocumentSymbols(root_nodes) # update cache """ @@ -1050,7 +1087,7 @@ class SolidLanguageServer(ABC): ) return [] else: - _, root_nodes = self.request_document_symbols(within_relative_path, include_body=include_body) + root_nodes = self.request_document_symbols(within_relative_path, include_body=include_body).root_symbols return root_nodes # Helper function to recursively process directories @@ -1109,7 +1146,8 @@ class SolidLanguageServer(ABC): child["parent"] = package_symbol elif os.path.isfile(contained_dir_or_file_abs_path): - _, file_root_nodes = self.request_document_symbols(contained_dir_or_file_rel_path, include_body=include_body) + document_symbols = self.request_document_symbols(contained_dir_or_file_rel_path, include_body=include_body) + file_root_nodes = document_symbols.root_symbols # Create file symbol, link with children file_rel_path = str(Path(contained_dir_or_file_abs_path).resolve().relative_to(self.repository_root_path)) @@ -1218,8 +1256,7 @@ class SolidLanguageServer(ABC): """ :return: the top-level symbols in the given file. """ - _, document_roots = self.request_document_symbols(relative_file_path) - return document_roots + return self.request_document_symbols(relative_file_path).root_symbols def request_overview(self, within_relative_path: str) -> dict[str, list[UnifiedSymbolInformation]]: """ @@ -1356,8 +1393,8 @@ class SolidLanguageServer(ABC): ref_text = file_data.contents.split("\n")[ref_line] if "." in ref_text: containing_symbol_name = ref_text.split(".")[0] - all_symbols, _ = self.request_document_symbols(ref_path) - for symbol in all_symbols: + document_symbols = self.request_document_symbols(ref_path) + for symbol in document_symbols.iter_symbols(): if symbol["name"] == containing_symbol_name and symbol["kind"] == ls_types.SymbolKind.Variable: containing_symbol = copy(symbol) containing_symbol["location"] = ref @@ -1478,12 +1515,12 @@ class SolidLanguageServer(ABC): ) return None - symbols, _ = self.request_document_symbols(relative_file_path) + document_symbols = self.request_document_symbols(relative_file_path) # make jedi and pyright api compatible # the former has no location, the later has no range # we will just always add location of the desired format to all symbols - for symbol in symbols: + for symbol in document_symbols.iter_symbols(): if "location" not in symbol: range = symbol["range"] location = ls_types.Location( @@ -1521,10 +1558,10 @@ class SolidLanguageServer(ABC): # Only consider containers that are not one-liners (otherwise we may get imports) candidate_containers = [ s - for s in symbols + for s in document_symbols.iter_symbols() if s["kind"] in container_symbol_kinds and s["location"]["range"]["start"]["line"] != s["location"]["range"]["end"]["line"] ] - var_containers = [s for s in symbols if s["kind"] == ls_types.SymbolKind.Variable] + var_containers = [s for s in document_symbols.iter_symbols() if s["kind"] == ls_types.SymbolKind.Variable] candidate_containers.extend(var_containers) if not candidate_containers: diff --git a/test/solidlsp/al/test_al_basic.py b/test/solidlsp/al/test_al_basic.py index 1fb95e42..47e859ca 100644 --- a/test/solidlsp/al/test_al_basic.py +++ b/test/solidlsp/al/test_al_basic.py @@ -43,7 +43,7 @@ class TestALLanguageServer: def test_find_table_fields(self, language_server: SolidLanguageServer) -> None: """Test that AL Language Server can find fields within a table.""" file_path = os.path.join("src", "Tables", "Customer.Table.al") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() # AL tables should have their fields as child symbols customer_table = None @@ -76,7 +76,7 @@ class TestALLanguageServer: def test_find_procedures(self, language_server: SolidLanguageServer) -> None: """Test that AL Language Server can find procedures in codeunits.""" file_path = os.path.join("src", "Codeunits", "CustomerMgt.Codeunit.al") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() # Find the codeunit symbol - AL returns 'Codeunit 50000 CustomerMgt' codeunit_symbol = None @@ -100,7 +100,7 @@ class TestALLanguageServer: """Test that AL Language Server can find references to symbols.""" # Find references to the Customer table from the CustomerMgt codeunit table_file = os.path.join("src", "Tables", "Customer.Table.al") - symbols = language_server.request_document_symbols(table_file) + symbols = language_server.request_document_symbols(table_file).get_all_symbols_and_roots() # Find the Customer table symbol customer_symbol = None diff --git a/test/solidlsp/bash/test_bash_basic.py b/test/solidlsp/bash/test_bash_basic.py index 4060f212..49bca382 100644 --- a/test/solidlsp/bash/test_bash_basic.py +++ b/test/solidlsp/bash/test_bash_basic.py @@ -25,7 +25,7 @@ class TestBashLanguageServerBasics: def test_bash_request_document_symbols(self, language_server: SolidLanguageServer) -> None: """Test request_document_symbols for bash files.""" # Test getting symbols from main.sh - all_symbols, _root_symbols = language_server.request_document_symbols("main.sh", include_body=False) + all_symbols, _root_symbols = language_server.request_document_symbols("main.sh", include_body=False).get_all_symbols_and_roots() # Extract function symbols (LSP Symbol Kind 12) function_symbols = [symbol for symbol in all_symbols if symbol.get("kind") == 12] @@ -41,7 +41,7 @@ class TestBashLanguageServerBasics: def test_bash_request_document_symbols_with_body(self, language_server: SolidLanguageServer) -> None: """Test request_document_symbols with body extraction.""" # Test with include_body=True - all_symbols, _root_symbols = language_server.request_document_symbols("main.sh", include_body=True) + all_symbols, _root_symbols = language_server.request_document_symbols("main.sh", include_body=True).get_all_symbols_and_roots() function_symbols = [symbol for symbol in all_symbols if symbol.get("kind") == 12] @@ -58,7 +58,9 @@ class TestBashLanguageServerBasics: def test_bash_utils_functions(self, language_server: SolidLanguageServer) -> None: """Test function detection in utils.sh file.""" # Test with utils.sh as well - utils_all_symbols, _utils_root_symbols = language_server.request_document_symbols("utils.sh", include_body=False) + utils_all_symbols, _utils_root_symbols = language_server.request_document_symbols( + "utils.sh", include_body=False + ).get_all_symbols_and_roots() utils_function_symbols = [symbol for symbol in utils_all_symbols if symbol.get("kind") == 12] utils_function_names = [symbol["name"] for symbol in utils_function_symbols] @@ -84,12 +86,16 @@ class TestBashLanguageServerBasics: def test_bash_function_syntax_patterns(self, language_server: SolidLanguageServer) -> None: """Test that LSP detects different bash function syntax patterns correctly.""" # Test main.sh (has both 'function' keyword and traditional syntax) - main_all_symbols, _main_root_symbols = language_server.request_document_symbols("main.sh", include_body=False) + main_all_symbols, _main_root_symbols = language_server.request_document_symbols( + "main.sh", include_body=False + ).get_all_symbols_and_roots() main_functions = [symbol for symbol in main_all_symbols if symbol.get("kind") == 12] main_function_names = [func["name"] for func in main_functions] # Test utils.sh (all use 'function' keyword) - utils_all_symbols, _utils_root_symbols = language_server.request_document_symbols("utils.sh", include_body=False) + utils_all_symbols, _utils_root_symbols = language_server.request_document_symbols( + "utils.sh", include_body=False + ).get_all_symbols_and_roots() utils_functions = [symbol for symbol in utils_all_symbols if symbol.get("kind") == 12] utils_function_names = [func["name"] for func in utils_functions] diff --git a/test/solidlsp/clojure/test_clojure_basic.py b/test/solidlsp/clojure/test_clojure_basic.py index dc10b885..c4037e54 100644 --- a/test/solidlsp/clojure/test_clojure_basic.py +++ b/test/solidlsp/clojure/test_clojure_basic.py @@ -53,7 +53,7 @@ class TestLanguageServerBasics: @pytest.mark.parametrize("language_server", [Language.CLOJURE], indirect=True) def test_document_symbols(self, language_server: SolidLanguageServer): - symbols, _ = language_server.request_document_symbols(CORE_PATH) + symbols, _ = language_server.request_document_symbols(CORE_PATH).get_all_symbols_and_roots() assert isinstance(symbols, list) and len(symbols) >= 4, "greet, add, multiply, -main functions" diff --git a/test/solidlsp/csharp/test_csharp_basic.py b/test/solidlsp/csharp/test_csharp_basic.py index 3c6d99c7..21ec4146 100644 --- a/test/solidlsp/csharp/test_csharp_basic.py +++ b/test/solidlsp/csharp/test_csharp_basic.py @@ -31,7 +31,7 @@ class TestCSharpLanguageServer: def test_get_document_symbols(self, language_server: SolidLanguageServer) -> None: """Test getting document symbols from a C# file.""" file_path = os.path.join("Program.cs") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() # Check that we have symbols assert len(symbols) > 0 @@ -49,7 +49,7 @@ class TestCSharpLanguageServer: def test_find_referencing_symbols(self, language_server: SolidLanguageServer) -> None: """Test finding references using symbol selection range.""" file_path = os.path.join("Program.cs") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() add_symbol = None # Handle nested symbol structure symbol_list = symbols[0] if symbols and isinstance(symbols[0], list) else symbols @@ -68,7 +68,7 @@ class TestCSharpLanguageServer: def test_nested_namespace_symbols(self, language_server: SolidLanguageServer) -> None: """Test getting symbols from nested namespace.""" file_path = os.path.join("Models", "Person.cs") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() # Check that we have symbols assert len(symbols) > 0 @@ -93,7 +93,7 @@ class TestCSharpLanguageServer: """Test finding references to Calculator.Subtract method across files.""" # First, find the Subtract method in Program.cs file_path = os.path.join("Program.cs") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() # Flatten the symbols if they're nested symbol_list = symbols[0] if symbols and isinstance(symbols[0], list) else symbols diff --git a/test/solidlsp/dart/test_dart_basic.py b/test/solidlsp/dart/test_dart_basic.py index 460f80c8..6ea1163a 100644 --- a/test/solidlsp/dart/test_dart_basic.py +++ b/test/solidlsp/dart/test_dart_basic.py @@ -171,7 +171,7 @@ class TestDartLanguageServer: def test_find_referencing_symbols(self, language_server: SolidLanguageServer) -> None: """Test finding references using symbol selection range.""" file_path = os.path.join("lib", "main.dart") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() # Handle nested symbol structure - symbols can be nested in lists symbol_list = symbols[0] if symbols and isinstance(symbols[0], list) else symbols @@ -281,7 +281,7 @@ class TestDartLanguageServer: def test_request_document_symbols(self, language_server: SolidLanguageServer) -> None: """Test getting document symbols from a Dart file.""" file_path = os.path.join("lib", "main.dart") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() # Check that we have symbols assert len(symbols) > 0 @@ -308,7 +308,7 @@ class TestDartLanguageServer: def test_request_referencing_symbols_comprehensive(self, language_server: SolidLanguageServer) -> None: """Test comprehensive referencing symbols functionality.""" file_path = os.path.join("lib", "main.dart") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() # Handle nested symbol structure symbol_list = symbols[0] if symbols and isinstance(symbols[0], list) else symbols @@ -339,7 +339,7 @@ class TestDartLanguageServer: helper_file_path = os.path.join("lib", "helper.dart") # Test finding references to subtract function from helper.dart in main.dart - helper_symbols = language_server.request_document_symbols(helper_file_path) + helper_symbols = language_server.request_document_symbols(helper_file_path).get_all_symbols_and_roots() symbol_list = helper_symbols[0] if helper_symbols and isinstance(helper_symbols[0], list) else helper_symbols subtract_symbol = next((s for s in symbol_list if s.get("name") == "subtract"), None) diff --git a/test/solidlsp/elixir/test_elixir_basic.py b/test/solidlsp/elixir/test_elixir_basic.py index 10391406..44bf7378 100644 --- a/test/solidlsp/elixir/test_elixir_basic.py +++ b/test/solidlsp/elixir/test_elixir_basic.py @@ -25,7 +25,7 @@ class TestElixirBasic: def test_request_references_function_definition(self, language_server: SolidLanguageServer): """Test finding references to a function definition.""" file_path = os.path.join("lib", "models.ex") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() # Find the User module's 'new' function user_new_symbol = None @@ -54,7 +54,7 @@ class TestElixirBasic: def test_request_references_create_user_function(self, language_server: SolidLanguageServer): """Test finding references to create_user function.""" file_path = os.path.join("lib", "services.ex") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() # Find the UserService module's 'create_user' function create_user_symbol = None @@ -79,7 +79,7 @@ class TestElixirBasic: def test_request_referencing_symbols_function(self, language_server: SolidLanguageServer): """Test finding symbols that reference a specific function.""" file_path = os.path.join("lib", "models.ex") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() # Find the User module's 'new' function user_new_symbol = None @@ -103,10 +103,10 @@ class TestElixirBasic: def test_timeout_enumeration_bug(self, language_server: SolidLanguageServer): """Test that enumeration doesn't timeout (regression test).""" # This should complete without timing out - symbols = language_server.request_document_symbols("lib/models.ex") + symbols = language_server.request_document_symbols("lib/models.ex").get_all_symbols_and_roots() assert symbols is not None # Test multiple symbol requests in succession for _ in range(3): - symbols = language_server.request_document_symbols("lib/services.ex") + symbols = language_server.request_document_symbols("lib/services.ex").get_all_symbols_and_roots() assert symbols is not None diff --git a/test/solidlsp/elixir/test_elixir_ignored_dirs.py b/test/solidlsp/elixir/test_elixir_ignored_dirs.py index 95a6f5f2..14f70dfd 100644 --- a/test/solidlsp/elixir/test_elixir_ignored_dirs.py +++ b/test/solidlsp/elixir/test_elixir_ignored_dirs.py @@ -46,7 +46,7 @@ def test_find_references_ignores_dir(ls_with_ignored_dirs: SolidLanguageServer): definition_file = "lib/models.ex" # Find the User struct definition - symbols = ls_with_ignored_dirs.request_document_symbols(definition_file) + symbols = ls_with_ignored_dirs.request_document_symbols(definition_file).get_all_symbols_and_roots() user_symbol = None for symbol_group in symbols: user_symbol = next((s for s in symbol_group if "User" in s.get("name", "")), None) @@ -87,7 +87,7 @@ def test_refs_and_symbols_with_glob_patterns(repo_path: Path) -> None: definition_file = "lib/models.ex" # Find the User struct definition - symbols = ls.request_document_symbols(definition_file) + symbols = ls.request_document_symbols(definition_file).get_all_symbols_and_roots() user_symbol = None for symbol_group in symbols: user_symbol = next((s for s in symbol_group if "User" in s.get("name", "")), None) diff --git a/test/solidlsp/elixir/test_elixir_integration.py b/test/solidlsp/elixir/test_elixir_integration.py index 8478332f..553d2569 100644 --- a/test/solidlsp/elixir/test_elixir_integration.py +++ b/test/solidlsp/elixir/test_elixir_integration.py @@ -72,7 +72,7 @@ class TestElixirIntegration: def test_module_hierarchy_understanding(self, language_server: SolidLanguageServer): """Test that the language server understands Elixir module hierarchy.""" models_file = os.path.join("lib", "models.ex") - symbols = language_server.request_document_symbols(models_file) + symbols = language_server.request_document_symbols(models_file).get_all_symbols_and_roots() if symbols: # Flatten symbol structure diff --git a/test/solidlsp/elixir/test_elixir_symbol_retrieval.py b/test/solidlsp/elixir/test_elixir_symbol_retrieval.py index a6c31b6d..b8c058cc 100644 --- a/test/solidlsp/elixir/test_elixir_symbol_retrieval.py +++ b/test/solidlsp/elixir/test_elixir_symbol_retrieval.py @@ -120,7 +120,7 @@ class TestElixirLanguageServerSymbols: # Test referencing symbols for User struct file_path = os.path.join("lib", "models.ex") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() user_symbol = None for symbol_group in symbols: user_symbol = next((s for s in symbol_group if "User" in s.get("name", "")), None) diff --git a/test/solidlsp/elm/test_elm_basic.py b/test/solidlsp/elm/test_elm_basic.py index 87669d8f..6283d914 100644 --- a/test/solidlsp/elm/test_elm_basic.py +++ b/test/solidlsp/elm/test_elm_basic.py @@ -20,7 +20,7 @@ class TestElmLanguageServer: @pytest.mark.parametrize("language_server", [Language.ELM], indirect=True) def test_find_references_within_file(self, language_server: SolidLanguageServer) -> None: file_path = os.path.join("Main.elm") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() greet_symbol = None for sym in symbols[0]: if sym.get("name") == "greet": @@ -35,7 +35,7 @@ class TestElmLanguageServer: def test_find_references_across_files(self, language_server: SolidLanguageServer) -> None: # Test formatMessage function which is defined in Utils.elm and used in Main.elm utils_path = os.path.join("Utils.elm") - symbols = language_server.request_document_symbols(utils_path) + symbols = language_server.request_document_symbols(utils_path).get_all_symbols_and_roots() formatMessage_symbol = None for sym in symbols[0]: if sym.get("name") == "formatMessage": diff --git a/test/solidlsp/erlang/test_erlang_basic.py b/test/solidlsp/erlang/test_erlang_basic.py index 6375e6bb..50ab3197 100644 --- a/test/solidlsp/erlang/test_erlang_basic.py +++ b/test/solidlsp/erlang/test_erlang_basic.py @@ -29,7 +29,7 @@ class TestErlangLanguageServerBasics: """Test document symbols retrieval for Erlang files.""" try: file_path = "hello.erl" - symbols_tuple = language_server.request_document_symbols(file_path) + symbols_tuple = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() assert isinstance(symbols_tuple, tuple) assert len(symbols_tuple) == 2 diff --git a/test/solidlsp/erlang/test_erlang_ignored_dirs.py b/test/solidlsp/erlang/test_erlang_ignored_dirs.py index bbb198a4..6b9ee69c 100644 --- a/test/solidlsp/erlang/test_erlang_ignored_dirs.py +++ b/test/solidlsp/erlang/test_erlang_ignored_dirs.py @@ -63,7 +63,7 @@ def test_find_references_ignores_dir(ls_with_ignored_dirs: SolidLanguageServer): definition_file = "include/records.hrl" # Find the user record definition - symbols = ls_with_ignored_dirs.request_document_symbols(definition_file) + symbols = ls_with_ignored_dirs.request_document_symbols(definition_file).get_all_symbols_and_roots() user_symbol = None for symbol_group in symbols: user_symbol = next((s for s in symbol_group if "user" in s.get("name", "").lower()), None) @@ -107,7 +107,7 @@ def test_refs_and_symbols_with_glob_patterns(repo_path: Path) -> None: definition_file = "include/records.hrl" # Find the user record definition - symbols = ls.request_document_symbols(definition_file) + symbols = ls.request_document_symbols(definition_file).get_all_symbols_and_roots() user_symbol = None for symbol_group in symbols: user_symbol = next((s for s in symbol_group if "user" in s.get("name", "").lower()), None) @@ -203,7 +203,7 @@ def test_document_symbols_ignores_dirs(ls_with_ignored_dirs: SolidLanguageServer # Try to get symbols from a file in ignored directory (should not find it) try: ignored_file = "ignored_dir/ignored_module.erl" - symbols = ls_with_ignored_dirs.request_document_symbols(ignored_file) + symbols = ls_with_ignored_dirs.request_document_symbols(ignored_file).get_all_symbols_and_roots() # If we get here, the file was found - symbols should be empty or None if symbols: assert len(symbols) == 0, "Should not find symbols in ignored directory" diff --git a/test/solidlsp/erlang/test_erlang_symbol_retrieval.py b/test/solidlsp/erlang/test_erlang_symbol_retrieval.py index f980853b..f423698f 100644 --- a/test/solidlsp/erlang/test_erlang_symbol_retrieval.py +++ b/test/solidlsp/erlang/test_erlang_symbol_retrieval.py @@ -126,7 +126,7 @@ class TestErlangLanguageServerSymbols: # Test referencing symbols for user record file_path = os.path.join("include", "records.hrl") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() user_symbol = None for symbol_group in symbols: user_symbol = next((s for s in symbol_group if "user" in s.get("name", "")), None) @@ -156,7 +156,7 @@ class TestErlangLanguageServerSymbols: # Test referencing symbols for create_user function file_path = os.path.join("src", "models.erl") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() create_user_symbol = None for symbol_group in symbols: create_user_symbol = next((s for s in symbol_group if "create_user" in s.get("name", "")), None) @@ -418,7 +418,7 @@ class TestErlangLanguageServerSymbols: # Test that we can find references to models module functions in services.erl file_path = os.path.join("src", "models.erl") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() create_user_symbol = None for symbol_group in symbols: create_user_symbol = next((s for s in symbol_group if "create_user" in s.get("name", "")), None) diff --git a/test/solidlsp/fortran/test_fortran_basic.py b/test/solidlsp/fortran/test_fortran_basic.py index 09b3dd20..e4ea280d 100644 --- a/test/solidlsp/fortran/test_fortran_basic.py +++ b/test/solidlsp/fortran/test_fortran_basic.py @@ -41,12 +41,12 @@ class TestFortranLanguageServer: def test_request_document_symbols(self, language_server: SolidLanguageServer) -> None: """Test that document symbols can be retrieved from Fortran files.""" # Test main.f90 - should have a program symbol - main_symbols, _ = language_server.request_document_symbols("main.f90") + main_symbols, _ = language_server.request_document_symbols("main.f90").get_all_symbols_and_roots() program_names = [s.get("name") for s in main_symbols] assert "test_program" in program_names, f"Program 'test_program' not found in main.f90. Found: {program_names}" # Test modules/math_utils.f90 - should have module and function symbols - module_symbols, _ = language_server.request_document_symbols("modules/math_utils.f90") + module_symbols, _ = language_server.request_document_symbols("modules/math_utils.f90").get_all_symbols_and_roots() all_names = [s.get("name") for s in module_symbols] assert "math_utils" in all_names, f"Module 'math_utils' not found. Found: {all_names}" assert "add_numbers" in all_names, f"Function 'add_numbers' not found. Found: {all_names}" @@ -60,7 +60,7 @@ class TestFortranLanguageServer: This tests the LSP textDocument/references capability. """ file_path = "modules/math_utils.f90" - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() # Find the add_numbers function add_numbers_symbol = None @@ -125,7 +125,7 @@ class TestFortranLanguageServer: """ # Get the add_numbers function symbol from math_utils.f90 file_path = "modules/math_utils.f90" - symbols, _ = language_server.request_document_symbols(file_path) + symbols, _ = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() # Find the add_numbers function add_numbers_symbol = None @@ -230,7 +230,7 @@ class TestFortranLanguageServer: fortls returns these as SymbolKind.Class (11) for types and SymbolKind.Interface (5) for interfaces. """ file_path = "modules/geometry.f90" - symbols, _ = language_server.request_document_symbols(file_path) + symbols, _ = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() # Find type and interface symbols type_names = [] diff --git a/test/solidlsp/go/test_go_basic.py b/test/solidlsp/go/test_go_basic.py index 56323157..eac511e9 100644 --- a/test/solidlsp/go/test_go_basic.py +++ b/test/solidlsp/go/test_go_basic.py @@ -19,7 +19,7 @@ class TestGoLanguageServer: @pytest.mark.parametrize("language_server", [Language.GO], indirect=True) def test_find_referencing_symbols(self, language_server: SolidLanguageServer) -> None: file_path = os.path.join("main.go") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() helper_symbol = None for sym in symbols[0]: if sym.get("name") == "Helper": diff --git a/test/solidlsp/haskell/test_haskell_basic.py b/test/solidlsp/haskell/test_haskell_basic.py index 062cc9ea..97820643 100644 --- a/test/solidlsp/haskell/test_haskell_basic.py +++ b/test/solidlsp/haskell/test_haskell_basic.py @@ -34,7 +34,7 @@ class TestHaskellLanguageServer: - All exported functions with correct names - Module structure """ - all_symbols, _ = language_server.request_document_symbols("src/Calculator.hs") + all_symbols, _ = language_server.request_document_symbols("src/Calculator.hs").get_all_symbols_and_roots() symbol_names = {s["name"] for s in all_symbols} # Verify exact set of expected top-level symbols @@ -71,7 +71,7 @@ class TestHaskellLanguageServer: Verifies Serena identifies all helper functions that are imported and used by Calculator module. """ - all_symbols, _ = language_server.request_document_symbols("src/Helper.hs") + all_symbols, _ = language_server.request_document_symbols("src/Helper.hs").get_all_symbols_and_roots() symbol_names = {s["name"] for s in all_symbols} # Verify expected helper functions (module name may also appear) @@ -97,7 +97,7 @@ class TestHaskellLanguageServer: Verifies Serena can identify cross-module dependencies. """ - all_symbols, _ = language_server.request_document_symbols("app/Main.hs") + all_symbols, _ = language_server.request_document_symbols("app/Main.hs").get_all_symbols_and_roots() symbol_names = {s["name"] for s in all_symbols} # Main.hs should have the main function diff --git a/test/solidlsp/java/test_java_basic.py b/test/solidlsp/java/test_java_basic.py index d74a9fea..0b081695 100644 --- a/test/solidlsp/java/test_java_basic.py +++ b/test/solidlsp/java/test_java_basic.py @@ -25,7 +25,7 @@ class TestJavaLanguageServer: # Dynamically determine the correct line/column for the 'Model' class name file_path = os.path.join("src", "main", "java", "test_repo", "Model.java") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() model_symbol = None for sym in symbols[0]: if sym.get("name") == "Model" and sym.get("kind") == 5: # 5 = Class diff --git a/test/solidlsp/julia/test_julia_basic.py b/test/solidlsp/julia/test_julia_basic.py index e411f3d8..99bf720c 100644 --- a/test/solidlsp/julia/test_julia_basic.py +++ b/test/solidlsp/julia/test_julia_basic.py @@ -11,7 +11,7 @@ class TestJuliaLanguageServer: """ Test if we can find the top-level symbols in the main.jl file. """ - all_symbols, _ = language_server.request_document_symbols("main.jl") + all_symbols, _ = language_server.request_document_symbols("main.jl").get_all_symbols_and_roots() symbol_names = {s["name"] for s in all_symbols} assert "calculate_sum" in symbol_names assert "main" in symbol_names diff --git a/test/solidlsp/kotlin/test_kotlin_basic.py b/test/solidlsp/kotlin/test_kotlin_basic.py index 6c421ab4..c3e209e0 100644 --- a/test/solidlsp/kotlin/test_kotlin_basic.py +++ b/test/solidlsp/kotlin/test_kotlin_basic.py @@ -25,7 +25,7 @@ class TestKotlinLanguageServer: # Dynamically determine the correct line/column for the 'Model' class name file_path = os.path.join("src", "main", "kotlin", "test_repo", "Model.kt") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() model_symbol = None for sym in symbols[0]: print(sym) diff --git a/test/solidlsp/lua/test_lua_basic.py b/test/solidlsp/lua/test_lua_basic.py index 66fa2074..4d4849cf 100644 --- a/test/solidlsp/lua/test_lua_basic.py +++ b/test/solidlsp/lua/test_lua_basic.py @@ -19,7 +19,7 @@ class TestLuaLanguageServer: @pytest.mark.parametrize("language_server", [Language.LUA], indirect=True) def test_find_symbols_in_calculator(self, language_server: SolidLanguageServer) -> None: """Test finding specific functions in calculator.lua.""" - symbols = language_server.request_document_symbols("src/calculator.lua") + symbols = language_server.request_document_symbols("src/calculator.lua").get_all_symbols_and_roots() assert symbols is not None assert len(symbols) > 0 @@ -49,7 +49,7 @@ class TestLuaLanguageServer: @pytest.mark.parametrize("language_server", [Language.LUA], indirect=True) def test_find_symbols_in_utils(self, language_server: SolidLanguageServer) -> None: """Test finding specific functions in utils.lua.""" - symbols = language_server.request_document_symbols("src/utils.lua") + symbols = language_server.request_document_symbols("src/utils.lua").get_all_symbols_and_roots() assert symbols is not None assert len(symbols) > 0 @@ -84,7 +84,7 @@ class TestLuaLanguageServer: @pytest.mark.parametrize("language_server", [Language.LUA], indirect=True) def test_find_symbols_in_main(self, language_server: SolidLanguageServer) -> None: """Test finding functions in main.lua.""" - symbols = language_server.request_document_symbols("main.lua") + symbols = language_server.request_document_symbols("main.lua").get_all_symbols_and_roots() assert symbols is not None assert len(symbols) > 0 @@ -107,7 +107,7 @@ class TestLuaLanguageServer: @pytest.mark.parametrize("language_server", [Language.LUA], indirect=True) def test_cross_file_references_calculator_add(self, language_server: SolidLanguageServer) -> None: """Test finding cross-file references to calculator.add function.""" - symbols = language_server.request_document_symbols("src/calculator.lua") + symbols = language_server.request_document_symbols("src/calculator.lua").get_all_symbols_and_roots() assert symbols is not None symbol_list = symbols[0] if isinstance(symbols, tuple) else symbols @@ -163,7 +163,7 @@ class TestLuaLanguageServer: @pytest.mark.parametrize("language_server", [Language.LUA], indirect=True) def test_cross_file_references_utils_trim(self, language_server: SolidLanguageServer) -> None: """Test finding cross-file references to utils.trim function.""" - symbols = language_server.request_document_symbols("src/utils.lua") + symbols = language_server.request_document_symbols("src/utils.lua").get_all_symbols_and_roots() assert symbols is not None symbol_list = symbols[0] if isinstance(symbols, tuple) else symbols @@ -245,7 +245,7 @@ class TestLuaLanguageServer: def test_references_between_test_and_source(self, language_server: SolidLanguageServer) -> None: """Test finding references from test files to source files.""" # Check if test_calculator.lua references calculator module - test_symbols = language_server.request_document_symbols("tests/test_calculator.lua") + test_symbols = language_server.request_document_symbols("tests/test_calculator.lua").get_all_symbols_and_roots() assert test_symbols is not None assert len(test_symbols) > 0 diff --git a/test/solidlsp/markdown/test_markdown_basic.py b/test/solidlsp/markdown/test_markdown_basic.py index 2f472638..2c9779da 100644 --- a/test/solidlsp/markdown/test_markdown_basic.py +++ b/test/solidlsp/markdown/test_markdown_basic.py @@ -25,7 +25,7 @@ class TestMarkdownLanguageServerBasics: def test_markdown_request_document_symbols(self, language_server: SolidLanguageServer) -> None: """Test request_document_symbols for markdown files.""" # Test getting symbols from README.md - all_symbols, _root_symbols = language_server.request_document_symbols("README.md", include_body=False) + all_symbols, _root_symbols = language_server.request_document_symbols("README.md", include_body=False).get_all_symbols_and_roots() # Extract heading symbols (LSP Symbol Kind 15 is String, but marksman uses kind 15 for headings) # Note: Different markdown LSPs may use different symbol kinds for headings @@ -38,7 +38,7 @@ class TestMarkdownLanguageServerBasics: @pytest.mark.parametrize("language_server", [Language.MARKDOWN], indirect=True) def test_markdown_request_symbols_from_guide(self, language_server: SolidLanguageServer) -> None: """Test symbol detection in guide.md file.""" - all_symbols, _root_symbols = language_server.request_document_symbols("guide.md", include_body=False) + all_symbols, _root_symbols = language_server.request_document_symbols("guide.md", include_body=False).get_all_symbols_and_roots() # At least some headings should be found assert len(all_symbols) > 0, f"Should find headings in guide.md, found {len(all_symbols)}" @@ -46,7 +46,7 @@ class TestMarkdownLanguageServerBasics: @pytest.mark.parametrize("language_server", [Language.MARKDOWN], indirect=True) def test_markdown_request_symbols_from_api(self, language_server: SolidLanguageServer) -> None: """Test symbol detection in api.md file.""" - all_symbols, _root_symbols = language_server.request_document_symbols("api.md", include_body=False) + all_symbols, _root_symbols = language_server.request_document_symbols("api.md", include_body=False).get_all_symbols_and_roots() # Should detect headings from api.md assert len(all_symbols) > 0, f"Should find headings in api.md, found {len(all_symbols)}" @@ -55,7 +55,7 @@ class TestMarkdownLanguageServerBasics: def test_markdown_request_document_symbols_with_body(self, language_server: SolidLanguageServer) -> None: """Test request_document_symbols with body extraction.""" # Test with include_body=True - all_symbols, _root_symbols = language_server.request_document_symbols("README.md", include_body=True) + all_symbols, _root_symbols = language_server.request_document_symbols("README.md", include_body=True).get_all_symbols_and_roots() # Should have found some symbols assert len(all_symbols) > 0, "Should find symbols in README.md" diff --git a/test/solidlsp/nix/test_nix_basic.py b/test/solidlsp/nix/test_nix_basic.py index 81a68669..d118904f 100644 --- a/test/solidlsp/nix/test_nix_basic.py +++ b/test/solidlsp/nix/test_nix_basic.py @@ -22,7 +22,7 @@ class TestNixLanguageServer: @pytest.mark.parametrize("language_server", [Language.NIX], indirect=True) def test_find_symbols_in_default_nix(self, language_server: SolidLanguageServer) -> None: """Test finding specific symbols in default.nix.""" - symbols = language_server.request_document_symbols("default.nix") + symbols = language_server.request_document_symbols("default.nix").get_all_symbols_and_roots() assert symbols is not None assert len(symbols) > 0 @@ -42,7 +42,7 @@ class TestNixLanguageServer: @pytest.mark.parametrize("language_server", [Language.NIX], indirect=True) def test_find_symbols_in_utils(self, language_server: SolidLanguageServer) -> None: """Test finding symbols in lib/utils.nix.""" - symbols = language_server.request_document_symbols("lib/utils.nix") + symbols = language_server.request_document_symbols("lib/utils.nix").get_all_symbols_and_roots() assert symbols is not None assert len(symbols) > 0 @@ -58,7 +58,7 @@ class TestNixLanguageServer: @pytest.mark.parametrize("language_server", [Language.NIX], indirect=True) def test_find_symbols_in_flake(self, language_server: SolidLanguageServer) -> None: """Test finding symbols in flake.nix.""" - symbols = language_server.request_document_symbols("flake.nix") + symbols = language_server.request_document_symbols("flake.nix").get_all_symbols_and_roots() assert symbols is not None assert len(symbols) > 0 @@ -72,7 +72,7 @@ class TestNixLanguageServer: @pytest.mark.parametrize("language_server", [Language.NIX], indirect=True) def test_find_symbols_in_module(self, language_server: SolidLanguageServer) -> None: """Test finding symbols in a NixOS module.""" - symbols = language_server.request_document_symbols("modules/example.nix") + symbols = language_server.request_document_symbols("modules/example.nix").get_all_symbols_and_roots() assert symbols is not None assert len(symbols) > 0 @@ -86,7 +86,7 @@ class TestNixLanguageServer: @pytest.mark.parametrize("language_server", [Language.NIX], indirect=True) def test_find_references_within_file(self, language_server: SolidLanguageServer) -> None: """Test finding references within the same file.""" - symbols = language_server.request_document_symbols("default.nix") + symbols = language_server.request_document_symbols("default.nix").get_all_symbols_and_roots() assert symbols is not None symbol_list = symbols[0] if isinstance(symbols, tuple) else symbols @@ -154,7 +154,7 @@ class TestNixLanguageServer: def test_verify_imports_exist(self, language_server: SolidLanguageServer) -> None: """Verify that our test files have proper imports set up.""" # Verify that default.nix imports utils from lib/utils.nix - symbols = language_server.request_document_symbols("default.nix") + symbols = language_server.request_document_symbols("default.nix").get_all_symbols_and_roots() assert symbols is not None symbol_list = symbols[0] if isinstance(symbols, tuple) else symbols @@ -164,7 +164,7 @@ class TestNixLanguageServer: assert "makeGreeting" in symbol_names, "makeGreeting should be found in default.nix" # Verify lib/utils.nix has the expected structure - utils_symbols = language_server.request_document_symbols("lib/utils.nix") + utils_symbols = language_server.request_document_symbols("lib/utils.nix").get_all_symbols_and_roots() assert utils_symbols is not None utils_list = utils_symbols[0] if isinstance(utils_symbols, tuple) else utils_symbols utils_names = {sym.get("name") for sym in utils_list if isinstance(sym, dict)} diff --git a/test/solidlsp/perl/test_perl_basic.py b/test/solidlsp/perl/test_perl_basic.py index b7b936fb..4c18627f 100644 --- a/test/solidlsp/perl/test_perl_basic.py +++ b/test/solidlsp/perl/test_perl_basic.py @@ -31,7 +31,7 @@ class TestPerlLanguageServer: def test_document_symbols(self, language_server: SolidLanguageServer) -> None: """Test that document symbols are correctly identified.""" # Request document symbols - all_symbols, _ = language_server.request_document_symbols("main.pl", include_body=False) + all_symbols, _ = language_server.request_document_symbols("main.pl", include_body=False).get_all_symbols_and_roots() assert all_symbols, "Expected to find symbols in main.pl" assert len(all_symbols) > 0, "Expected at least one symbol" diff --git a/test/solidlsp/python/test_python_basic.py b/test/solidlsp/python/test_python_basic.py index dbfd8ce4..8dafc964 100644 --- a/test/solidlsp/python/test_python_basic.py +++ b/test/solidlsp/python/test_python_basic.py @@ -26,7 +26,7 @@ class TestLanguageServerBasics: file_path = os.path.join("test_repo", "models.py") # Line 31 contains the User class definition # Use selectionRange only - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() user_symbol = next((s for s in symbols[0] if s.get("name") == "User"), None) if not user_symbol or "selectionRange" not in user_symbol: raise AssertionError("User symbol or its selectionRange not found") @@ -41,7 +41,7 @@ class TestLanguageServerBasics: file_path = os.path.join("test_repo", "models.py") # Line 56 contains the Item class definition # Use selectionRange only - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() item_symbol = next((s for s in symbols[0] if s.get("name") == "Item"), None) if not item_symbol or "selectionRange" not in item_symbol: raise AssertionError("Item symbol or its selectionRange not found") @@ -57,7 +57,7 @@ class TestLanguageServerBasics: file_path = os.path.join("test_repo", "services.py") # Line 24 contains the get_user method with id parameter # Use selectionRange only - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() get_user_symbol = next((s for s in symbols[0] if s.get("name") == "get_user"), None) if not get_user_symbol or "selectionRange" not in get_user_symbol: raise AssertionError("get_user symbol or its selectionRange not found") @@ -71,7 +71,7 @@ class TestLanguageServerBasics: file_path = os.path.join("test_repo", "services.py") # Line 15 contains the create_user method definition # Use selectionRange only - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() create_user_symbol = next((s for s in symbols[0] if s.get("name") == "create_user"), None) if not create_user_symbol or "selectionRange" not in create_user_symbol: raise AssertionError("create_user symbol or its selectionRange not found") diff --git a/test/solidlsp/python/test_symbol_retrieval.py b/test/solidlsp/python/test_symbol_retrieval.py index 931cb20e..6875bc89 100644 --- a/test/solidlsp/python/test_symbol_retrieval.py +++ b/test/solidlsp/python/test_symbol_retrieval.py @@ -107,7 +107,7 @@ class TestLanguageServerSymbols: # Test referencing symbols for create_user function file_path = os.path.join("test_repo", "services.py") # Line 15 contains the create_user function definition - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() create_user_symbol = next((s for s in symbols[0] if s.get("name") == "create_user"), None) if not create_user_symbol or "selectionRange" not in create_user_symbol: raise AssertionError("create_user symbol or its selectionRange not found") @@ -131,7 +131,7 @@ class TestLanguageServerSymbols: # Test referencing symbols for User class file_path = os.path.join("test_repo", "models.py") # Line 31 contains the User class definition - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() user_symbol = next((s for s in symbols[0] if s.get("name") == "User"), None) if not user_symbol or "selectionRange" not in user_symbol: raise AssertionError("User symbol or its selectionRange not found") @@ -152,7 +152,7 @@ class TestLanguageServerSymbols: # Test referencing symbols for id parameter in get_user file_path = os.path.join("test_repo", "services.py") # Line 24 contains the get_user method with id parameter - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() get_user_symbol = next((s for s in symbols[0] if s.get("name") == "get_user"), None) if not get_user_symbol or "selectionRange" not in get_user_symbol: raise AssertionError("get_user symbol or its selectionRange not found") @@ -390,7 +390,9 @@ class TestLanguageServerSymbols: if "location" in user_management_node and "relativePath" in user_management_node["location"]: user_management_rel_path = user_management_node["location"]["relativePath"] assert user_management_rel_path == os.path.join("examples", "user_management.py") - _, user_management_roots = language_server.request_document_symbols(os.path.join("examples", "user_management.py")) + _, user_management_roots = language_server.request_document_symbols( + os.path.join("examples", "user_management.py") + ).get_all_symbols_and_roots() assert user_management_roots == user_management_node["children"] @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) @@ -411,7 +413,9 @@ class TestLanguageServerSymbols: if "location" in user_management_node and "relativePath" in user_management_node["location"]: user_management_rel_path = user_management_node["location"]["relativePath"] assert user_management_rel_path == os.path.join("examples", "user_management.py") - _, user_management_roots = language_server.request_document_symbols(os.path.join("examples", "user_management.py")) + _, user_management_roots = language_server.request_document_symbols( + os.path.join("examples", "user_management.py") + ).get_all_symbols_and_roots() assert user_management_roots == user_management_node["children"] @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) diff --git a/test/solidlsp/r/test_r_basic.py b/test/solidlsp/r/test_r_basic.py index fc01d58c..3ce63a0b 100644 --- a/test/solidlsp/r/test_r_basic.py +++ b/test/solidlsp/r/test_r_basic.py @@ -27,7 +27,7 @@ class TestRLanguageServer: @pytest.mark.parametrize("language_server", [Language.R], indirect=True) def test_symbol_retrieval(self, language_server: SolidLanguageServer): """Test R document symbol extraction.""" - all_symbols, _root_symbols = language_server.request_document_symbols(os.path.join("R", "utils.R")) + all_symbols, _root_symbols = language_server.request_document_symbols(os.path.join("R", "utils.R")).get_all_symbols_and_roots() # Should find the three exported functions function_symbols = [s for s in all_symbols if s.get("kind") == 12] # Function kind diff --git a/test/solidlsp/rego/test_rego_basic.py b/test/solidlsp/rego/test_rego_basic.py index 814562dc..e4185e85 100644 --- a/test/solidlsp/rego/test_rego_basic.py +++ b/test/solidlsp/rego/test_rego_basic.py @@ -21,7 +21,7 @@ class TestRegoLanguageServer: def test_request_document_symbols_authz(self, language_server: SolidLanguageServer) -> None: """Test that document symbols can be retrieved from authz.rego.""" file_path = os.path.join("policies", "authz.rego") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() assert symbols is not None assert len(symbols) > 0 @@ -40,7 +40,7 @@ class TestRegoLanguageServer: def test_request_document_symbols_helpers(self, language_server: SolidLanguageServer) -> None: """Test that document symbols can be retrieved from helpers.rego.""" file_path = os.path.join("utils", "helpers.rego") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() assert symbols is not None assert len(symbols) > 0 @@ -71,7 +71,7 @@ class TestRegoLanguageServer: file_path = os.path.join("policies", "authz.rego") # Get document symbols - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() symbol_list = symbols[0] if isinstance(symbols, tuple) else symbols # Find the is_admin symbol which references admin_roles @@ -98,7 +98,7 @@ class TestRegoLanguageServer: file_path = os.path.join("policies", "authz.rego") # Get document symbols - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() symbol_list = symbols[0] if isinstance(symbols, tuple) else symbols # Find the allow symbol @@ -123,7 +123,7 @@ class TestRegoLanguageServer: def test_find_symbols_validation(self, language_server: SolidLanguageServer) -> None: """Test finding symbols in validation.rego which has imports.""" file_path = os.path.join("policies", "validation.rego") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() assert symbols is not None assert len(symbols) > 0 diff --git a/test/solidlsp/ruby/test_ruby_basic.py b/test/solidlsp/ruby/test_ruby_basic.py index 2edf95f2..b7a4cf75 100644 --- a/test/solidlsp/ruby/test_ruby_basic.py +++ b/test/solidlsp/ruby/test_ruby_basic.py @@ -20,7 +20,7 @@ class TestRubyLanguageServer: @pytest.mark.parametrize("language_server", [Language.RUBY], indirect=True) def test_find_referencing_symbols(self, language_server: SolidLanguageServer) -> None: file_path = os.path.join("main.rb") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() helper_symbol = None for sym in symbols[0]: if sym.get("name") == "helper_function": diff --git a/test/solidlsp/ruby/test_ruby_symbol_retrieval.py b/test/solidlsp/ruby/test_ruby_symbol_retrieval.py index 9ac4133e..ae089830 100644 --- a/test/solidlsp/ruby/test_ruby_symbol_retrieval.py +++ b/test/solidlsp/ruby/test_ruby_symbol_retrieval.py @@ -89,7 +89,7 @@ class TestRubyLanguageServerSymbols: """Test request_containing_symbol for a module context.""" # Test that we can find the Services module in document symbols file_path = os.path.join("services.rb") - symbols, _roots = language_server.request_document_symbols(file_path) + symbols, _roots = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() # Verify Services module appears in document symbols services_module = None @@ -141,7 +141,7 @@ class TestRubyLanguageServerSymbols: # Test referencing symbols for create_user method file_path = os.path.join("services.rb") # Line containing the create_user method definition - symbols, _roots = language_server.request_document_symbols(file_path) + symbols, _roots = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() create_user_symbol = None # Find create_user method in the document symbols (Ruby returns flat list) @@ -169,7 +169,7 @@ class TestRubyLanguageServerSymbols: # Test referencing symbols for User class file_path = os.path.join("models.rb") # Find User class in document symbols - symbols, _roots = language_server.request_document_symbols(file_path) + symbols, _roots = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() user_symbol = None for symbol in symbols: @@ -285,7 +285,7 @@ class TestRubyLanguageServerSymbols: def test_document_symbols_detailed(self, language_server: SolidLanguageServer) -> None: """Test document symbols for detailed Ruby file structure.""" file_path = os.path.join("models.rb") - symbols, roots = language_server.request_document_symbols(file_path) + symbols, roots = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() # Verify we have symbols assert len(symbols) > 0 or len(roots) > 0 @@ -310,7 +310,7 @@ class TestRubyLanguageServerSymbols: def test_module_and_class_hierarchy(self, language_server: SolidLanguageServer) -> None: """Test symbol detection for modules and nested class hierarchies.""" file_path = os.path.join("nested.rb") - symbols, roots = language_server.request_document_symbols(file_path) + symbols, roots = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() # Verify we can detect the nested structure assert len(symbols) > 0 or len(roots) > 0 @@ -363,7 +363,7 @@ class TestRubyLanguageServerSymbols: # Test referencing symbols for a method parameter in get_user method file_path = os.path.join("services.rb") # Find get_user method and test parameter references - symbols, _roots = language_server.request_document_symbols(file_path) + symbols, _roots = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() get_user_symbol = None for symbol in symbols: diff --git a/test/solidlsp/rust/test_rust_2024_edition.py b/test/solidlsp/rust/test_rust_2024_edition.py index e73c8d6d..1facfd28 100644 --- a/test/solidlsp/rust/test_rust_2024_edition.py +++ b/test/solidlsp/rust/test_rust_2024_edition.py @@ -31,7 +31,7 @@ class TestRust2024EditionLanguageServer: def test_find_references_raw(self) -> None: # Test finding references to the 'add' function defined in main.rs file_path = os.path.join("src", "main.rs") - symbols = self.language_server.request_document_symbols(file_path) + symbols = self.language_server.request_document_symbols(file_path).get_all_symbols_and_roots() add_symbol = None for sym in symbols[0]: if sym.get("name") == "add": @@ -53,7 +53,7 @@ class TestRust2024EditionLanguageServer: def test_find_referencing_symbols_multiply(self) -> None: # Find references to 'multiply' function defined in lib.rs file_path = os.path.join("src", "lib.rs") - symbols = self.language_server.request_document_symbols(file_path) + symbols = self.language_server.request_document_symbols(file_path).get_all_symbols_and_roots() multiply_symbol = None for sym in symbols[0]: if sym.get("name") == "multiply": @@ -69,7 +69,7 @@ class TestRust2024EditionLanguageServer: def test_find_calculator_struct_and_impl(self) -> None: # Test finding the Calculator struct and its impl block file_path = os.path.join("src", "lib.rs") - symbols = self.language_server.request_document_symbols(file_path) + symbols = self.language_server.request_document_symbols(file_path).get_all_symbols_and_roots() # Find the Calculator struct calculator_struct = None diff --git a/test/solidlsp/rust/test_rust_basic.py b/test/solidlsp/rust/test_rust_basic.py index d0576de2..c827d4b2 100644 --- a/test/solidlsp/rust/test_rust_basic.py +++ b/test/solidlsp/rust/test_rust_basic.py @@ -13,7 +13,7 @@ class TestRustLanguageServer: def test_find_references_raw(self, language_server: SolidLanguageServer) -> None: # Directly test the request_references method for the add function file_path = os.path.join("src", "lib.rs") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() add_symbol = None for sym in symbols[0]: if sym.get("name") == "add": @@ -37,7 +37,7 @@ class TestRustLanguageServer: def test_find_referencing_symbols(self, language_server: SolidLanguageServer) -> None: # Find references to 'add' defined in lib.rs, should be referenced from main.rs file_path = os.path.join("src", "lib.rs") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() add_symbol = None for sym in symbols[0]: if sym.get("name") == "add": diff --git a/test/solidlsp/scala/test_scala_language_server.py b/test/solidlsp/scala/test_scala_language_server.py index 2b7f4bea..f22e2e96 100644 --- a/test/solidlsp/scala/test_scala_language_server.py +++ b/test/solidlsp/scala/test_scala_language_server.py @@ -29,7 +29,7 @@ def scala_ls(): def test_scala_document_symbols(scala_ls): """Test document symbols for Main.scala""" - symbols, _ = scala_ls.request_document_symbols(MAIN_FILE_PATH) + symbols, _ = scala_ls.request_document_symbols(MAIN_FILE_PATH).get_all_symbols_and_roots() symbol_names = [s["name"] for s in symbols] assert symbol_names[0] == "com.example" assert symbol_names[1] == "Main" diff --git a/test/solidlsp/swift/test_swift_basic.py b/test/solidlsp/swift/test_swift_basic.py index 67dfe686..5cd5616c 100644 --- a/test/solidlsp/swift/test_swift_basic.py +++ b/test/solidlsp/swift/test_swift_basic.py @@ -88,7 +88,7 @@ class TestSwiftLanguageServerBasics: # First, let's check if Utils is used anywhere (it might not be in this simple test) # We'll test goto_definition on Utils struct itself - symbols = language_server.request_document_symbols(utils_file) + symbols = language_server.request_document_symbols(utils_file).get_all_symbols_and_roots() utils_symbol = next((s for s in symbols[0] if s.get("name") == "Utils"), None) sel_start = utils_symbol["selectionRange"]["start"] @@ -104,7 +104,7 @@ class TestSwiftLanguageServerBasics: """Test request_references on the Calculator class.""" # Get references to the Calculator class in main.swift file_path = os.path.join("src", "main.swift") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() calculator_symbol = next((s for s in symbols[0] if s.get("name") == "Calculator"), None) @@ -126,7 +126,7 @@ class TestSwiftLanguageServerBasics: """Test request_references on the User struct.""" # Get references to the User struct in main.swift file_path = os.path.join("src", "main.swift") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() user_symbol = next((s for s in symbols[0] if s.get("name") == "User"), None) @@ -147,7 +147,7 @@ class TestSwiftLanguageServerBasics: """Test request_references on the Utils struct.""" # Get references to the Utils struct in utils.swift file_path = os.path.join("src", "utils.swift") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() utils_symbol = next((s for s in symbols[0] if s.get("name") == "Utils"), None) if not utils_symbol or "selectionRange" not in utils_symbol: raise AssertionError("Utils symbol or its selectionRange not found") diff --git a/test/solidlsp/terraform/test_terraform_basic.py b/test/solidlsp/terraform/test_terraform_basic.py index ff20ac66..d608e4e5 100644 --- a/test/solidlsp/terraform/test_terraform_basic.py +++ b/test/solidlsp/terraform/test_terraform_basic.py @@ -21,7 +21,7 @@ class TestLanguageServerBasics: # Simple test to verify the language server is working file_path = "main.tf" # Just try to get document symbols - this should work without hanging - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() assert len(symbols) > 0, "Should find at least some symbols in main.tf" @pytest.mark.parametrize("language_server", [Language.TERRAFORM], indirect=True) @@ -30,7 +30,7 @@ class TestLanguageServerBasics: # Get references to an aws_instance resource in main.tf file_path = "main.tf" # Find aws_instance resources - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() aws_instance_symbol = next((s for s in symbols[0] if s.get("name") == 'resource "aws_instance" "web_server"'), None) if not aws_instance_symbol or "selectionRange" not in aws_instance_symbol: raise AssertionError("aws_instance symbol or its selectionRange not found") @@ -44,7 +44,7 @@ class TestLanguageServerBasics: # Get references to a variable in variables.tf file_path = "variables.tf" # Find variable definitions - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() var_symbol = next((s for s in symbols[0] if s.get("name") == 'variable "instance_type"'), None) if not var_symbol or "selectionRange" not in var_symbol: raise AssertionError("variable symbol or its selectionRange not found") diff --git a/test/solidlsp/typescript/test_typescript_basic.py b/test/solidlsp/typescript/test_typescript_basic.py index 1410c5c4..9570bdc2 100644 --- a/test/solidlsp/typescript/test_typescript_basic.py +++ b/test/solidlsp/typescript/test_typescript_basic.py @@ -19,7 +19,7 @@ class TestTypescriptLanguageServer: @pytest.mark.parametrize("language_server", [Language.TYPESCRIPT], indirect=True) def test_find_referencing_symbols(self, language_server: SolidLanguageServer) -> None: file_path = os.path.join("index.ts") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() helper_symbol = None for sym in symbols[0]: if sym.get("name") == "helperFunction": diff --git a/test/solidlsp/zig/test_zig_basic.py b/test/solidlsp/zig/test_zig_basic.py index 80f8fcc1..3753621f 100644 --- a/test/solidlsp/zig/test_zig_basic.py +++ b/test/solidlsp/zig/test_zig_basic.py @@ -30,7 +30,7 @@ class TestZigLanguageServer: def test_find_symbols_in_main(self, language_server: SolidLanguageServer) -> None: """Test finding specific symbols in main.zig.""" file_path = os.path.join("src", "main.zig") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() assert symbols is not None assert len(symbols) > 0 @@ -47,7 +47,7 @@ class TestZigLanguageServer: def test_find_symbols_in_calculator(self, language_server: SolidLanguageServer) -> None: """Test finding Calculator struct and its methods.""" file_path = os.path.join("src", "calculator.zig") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() assert symbols is not None assert len(symbols) > 0 @@ -90,7 +90,7 @@ class TestZigLanguageServer: def test_find_symbols_in_math_utils(self, language_server: SolidLanguageServer) -> None: """Test finding functions in math_utils.zig.""" file_path = os.path.join("src", "math_utils.zig") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() assert symbols is not None assert len(symbols) > 0 @@ -106,7 +106,7 @@ class TestZigLanguageServer: def test_find_references_within_file(self, language_server: SolidLanguageServer) -> None: """Test finding references within the same file.""" file_path = os.path.join("src", "calculator.zig") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() symbol_list = symbols[0] if isinstance(symbols, tuple) else symbols @@ -163,7 +163,7 @@ class TestZigLanguageServer: time.sleep(1) # Find Calculator struct - symbols = language_server.request_document_symbols(os.path.join("src", "calculator.zig")) + symbols = language_server.request_document_symbols(os.path.join("src", "calculator.zig")).get_all_symbols_and_roots() symbol_list = symbols[0] if isinstance(symbols, tuple) else symbols calculator_symbol = None @@ -207,7 +207,7 @@ class TestZigLanguageServer: """ # Find references to Calculator from calculator.zig file_path = os.path.join("src", "calculator.zig") - symbols = language_server.request_document_symbols(file_path) + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() symbol_list = symbols[0] if isinstance(symbols, tuple) else symbols calculator_symbol = None @@ -291,7 +291,7 @@ class TestZigLanguageServer: def test_verify_cross_file_imports(self, language_server: SolidLanguageServer) -> None: """Verify that our test files have proper cross-file imports.""" # Verify main.zig imports - main_symbols = language_server.request_document_symbols(os.path.join("src", "main.zig")) + main_symbols = language_server.request_document_symbols(os.path.join("src", "main.zig")).get_all_symbols_and_roots() assert main_symbols is not None main_list = main_symbols[0] if isinstance(main_symbols, tuple) else main_symbols main_names = {sym.get("name") for sym in main_list if isinstance(sym, dict)} @@ -301,14 +301,14 @@ class TestZigLanguageServer: assert "greeting" in main_names, "greeting function should be in main.zig" # Verify calculator.zig exports Calculator - calc_symbols = language_server.request_document_symbols(os.path.join("src", "calculator.zig")) + calc_symbols = language_server.request_document_symbols(os.path.join("src", "calculator.zig")).get_all_symbols_and_roots() assert calc_symbols is not None calc_list = calc_symbols[0] if isinstance(calc_symbols, tuple) else calc_symbols calc_names = {sym.get("name") for sym in calc_list if isinstance(sym, dict)} assert "Calculator" in calc_names, "Calculator struct should be in calculator.zig" # Verify math_utils.zig exports functions - math_symbols = language_server.request_document_symbols(os.path.join("src", "math_utils.zig")) + math_symbols = language_server.request_document_symbols(os.path.join("src", "math_utils.zig")).get_all_symbols_and_roots() assert math_symbols is not None math_list = math_symbols[0] if isinstance(math_symbols, tuple) else math_symbols math_names = {sym.get("name") for sym in math_list if isinstance(sym, dict)} From 7af490d0a20b162f18bdbee1be0ee9aa497f4b89 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 11 Nov 2025 20:40:03 +0100 Subject: [PATCH 14/26] Reintroduce (high-level) document symbols cache, now storing instance of DocumentSymbols --- src/solidlsp/ls.py | 139 +++++++++++++++++++++++++++++---------------- 1 file changed, 89 insertions(+), 50 deletions(-) diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index 641b355a..90b04e43 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -127,10 +127,14 @@ class SolidLanguageServer(ABC): """ CACHE_FOLDER_NAME = "cache" - CACHE_VERSION_RAW_DOCUMENT_SYMBOLS = 1 - CACHE_FILE_RAW_DOCUMENT_SYMBOLS = "raw_document_symbols.pkl" - CACHE_FILE_RAW_DOCUMENT_SYMBOLS_LEGACY = "document_symbols_cache_v23-06-25.pkl" - CACHE_VERSION = 2 + RAW_DOCUMENT_SYMBOLS_CACHE_VERSION = 1 + """ + global version identifier for raw symbol caches; an LS-specific version is defined separately and combined with this + """ + RAW_DOCUMENT_SYMBOL_CACHE_FILENAME = "raw_document_symbols.pkl" + RAW_DOCUMENT_SYMBOL_CACHE_FILENAME_LEGACY_FALLBACK = "document_symbols_cache_v23-06-25.pkl" + DOCUMENT_SYMBOL_CACHE_VERSION = 3 + DOCUMENT_SYMBOL_CACHE_FILENAME = "document_symbols.pkl" # To be overridden and extended by subclasses def is_ignored_dirname(self, dirname: str) -> bool: @@ -239,16 +243,22 @@ class SolidLanguageServer(ABC): self.open_file_buffers: dict[str, LSPFileBuffer] = {} self.language = Language(language_id) - # load caches + # initialise symbol caches self._cache_dir = ( Path(self.repository_root_path) / self._solidlsp_settings.project_data_relative_path / self.CACHE_FOLDER_NAME / self.language_id ) self._cache_dir.mkdir(parents=True, exist_ok=True) - self._raw_document_symbols_cache_version = cache_version_raw_document_symbols + # * raw document symbols cache + self._ls_specific_raw_document_symbols_cache_version = cache_version_raw_document_symbols self._raw_document_symbols_cache: dict[str, tuple[str, list[GenericDocumentSymbol]]] = {} - """Maps relative file paths to a tuple of (file_content_hash, root_symbols)""" + """maps relative file paths to a tuple of (file_content_hash, raw_root_symbols)""" self._raw_document_symbols_cache_is_modified: bool = False self._load_raw_document_symbols_cache() + # * high-level document symbols cache + self._document_symbols_cache: dict[str, tuple[str, DocumentSymbols]] = {} + """maps relative file paths to a tuple of (file_content_hash, document_symbols)""" + self._document_symbols_cache_is_modified: bool = False + self._load_document_symbols_cache() self.server_started = False self.completions_available = threading.Event() @@ -898,12 +908,12 @@ class SolidLanguageServer(ABC): if file_hash_and_result is not None: file_hash, result = file_hash_and_result if file_hash == fd.content_hash: - log.debug("Returning cached document symbols for %s", relative_file_path) + log.debug("Returning cached raw document symbols for %s", relative_file_path) return result else: - log.debug("Content for %s has changed. Will overwrite in-memory cache", relative_file_path) + log.debug("Raw document symbol content for %s has changed. Will overwrite in-memory cache", relative_file_path) else: - log.debug("No cache hit for symbols in %s", relative_file_path) + log.debug("No cache hit for raw document symbols symbols in %s", relative_file_path) return None def get_symbols(fd: LSPFileBuffer) -> list[GenericDocumentSymbol]: @@ -945,38 +955,32 @@ class SolidLanguageServer(ABC): If you need a symbol tree that contains file symbols as well, you should use `request_full_symbol_tree` instead. """ # check if the desired result is cached + cache_key = relative_file_path with self.open_file(relative_file_path) as file_data: - """ - cache_key = f"{relative_file_path}-{include_body}" - with self._cache_lock: - file_hash_and_result = self._raw_document_symbols_cache.get(cache_key) - if file_hash_and_result is not None: - file_hash, result = file_hash_and_result - if file_hash == file_data.content_hash: - self.logger.log(f"Returning cached document symbols for {relative_file_path}", logging.DEBUG) - return result - else: - self.logger.log(f"Content for {relative_file_path} has changed. Will overwrite in-memory cache", logging.DEBUG) + file_hash_and_result = self._document_symbols_cache.get(cache_key) + if file_hash_and_result is not None: + file_hash, document_symbols = file_hash_and_result + if file_hash == file_data.content_hash: + log.debug("Returning cached document symbols for %s", relative_file_path) + return document_symbols else: - self.logger.log(f"No cache hit for symbols with {include_body=} in {relative_file_path}", logging.DEBUG) - """ + log.debug("Cached document symbol content for %s has changed", relative_file_path) + else: + log.debug("No cache hit for document symbols in %s", relative_file_path) - # request the symbols from the language server + # no cached result: request the root symbols from the language server response = self._request_document_symbols(relative_file_path, file_data) if response is None: - self.logger.log( + log.warning( f"Received None response from the Language Server for document symbols in {relative_file_path}. " f"This means the language server can't understand this file (possibly due to syntax errors). It may also be due to a bug or misconfiguration of the LS. " f"Returning empty list", - logging.WARNING, ) - return [], [] + return DocumentSymbols([]) + assert isinstance(response, list), f"Unexpected response from Language Server: {response}" - self.logger.log( - f"Received {len(response)} document symbols for {relative_file_path} from the Language Server", - logging.DEBUG, - ) + log.debug("Received %d root symbols for %s from the language server", len(response), relative_file_path) def turn_item_into_unified_symbol(item: GenericDocumentSymbol) -> ls_types.UnifiedSymbolInformation: """ @@ -1042,17 +1046,14 @@ class SolidLanguageServer(ABC): root_nodes: list[ls_types.UnifiedSymbolInformation] = response convert_nodes_with_common_parent(root_nodes, None) - result = DocumentSymbols(root_nodes) + document_symbols = DocumentSymbols(root_nodes) # update cache - """ self.logger.log(f"Caching document symbols for {relative_file_path}", logging.DEBUG) - with self._cache_lock: - self._raw_document_symbols_cache[cache_key] = (file_data.content_hash, result) - self._raw_document_symbols_cache_has_changed = True - """ + self._document_symbols_cache[cache_key] = (file_data.content_hash, document_symbols) + self._document_symbols_cache_is_modified = True - return result + return document_symbols def request_full_symbol_tree( self, within_relative_path: str | None = None, include_body: bool = False @@ -1316,6 +1317,7 @@ class SolidLanguageServer(ABC): symbol_start_line = symbol["location"]["range"]["start"]["line"] symbol_end_line = symbol["location"]["range"]["end"]["line"] assert "relativePath" in symbol["location"] + # TODO fixme: duplicate read symbol_file = self.retrieve_full_file_content(symbol["location"]["relativePath"]) symbol_lines = symbol_file.split("\n") symbol_body = "\n".join(symbol_lines[symbol_start_line : symbol_end_line + 1]) @@ -1649,8 +1651,7 @@ class SolidLanguageServer(ABC): return defining_symbol def _save_raw_document_symbols_cache(self): - cache_file = self._cache_dir / self.CACHE_FILE_RAW_DOCUMENT_SYMBOLS - cache_version = (self.CACHE_VERSION_RAW_DOCUMENT_SYMBOLS, self._raw_document_symbols_cache_version) + cache_file = self._cache_dir / self.RAW_DOCUMENT_SYMBOL_CACHE_FILENAME if not self._raw_document_symbols_cache_is_modified: log.debug("No changes to raw document symbols cache, skipping save") @@ -1658,7 +1659,7 @@ class SolidLanguageServer(ABC): log.info("Saving updated raw document symbols cache to %s", cache_file) try: - save_cache(str(cache_file), cache_version, self._raw_document_symbols_cache) + save_cache(str(cache_file), self._raw_document_symbols_cache_version(), self._raw_document_symbols_cache) self._raw_document_symbols_cache_is_modified = False except Exception as e: log.error( @@ -1667,16 +1668,15 @@ class SolidLanguageServer(ABC): e, ) - def save_cache(self): - self._save_raw_document_symbols_cache() + def _raw_document_symbols_cache_version(self): + return (self.RAW_DOCUMENT_SYMBOLS_CACHE_VERSION, self._ls_specific_raw_document_symbols_cache_version) def _load_raw_document_symbols_cache(self): - cache_file = self._cache_dir / self.CACHE_FILE_RAW_DOCUMENT_SYMBOLS - cache_version = (self.CACHE_VERSION_RAW_DOCUMENT_SYMBOLS, self._raw_document_symbols_cache_version) + cache_file = self._cache_dir / self.RAW_DOCUMENT_SYMBOL_CACHE_FILENAME if not cache_file.exists(): # check for legacy cache to load to migrate - legacy_cache_file = self._cache_dir / self.CACHE_FILE_RAW_DOCUMENT_SYMBOLS_LEGACY + legacy_cache_file = self._cache_dir / self.RAW_DOCUMENT_SYMBOL_CACHE_FILENAME_LEGACY_FALLBACK if legacy_cache_file.exists(): try: legacy_cache: dict[ @@ -1698,20 +1698,59 @@ class SolidLanguageServer(ABC): return # cache file exists, so load it - log.info(f"Loading document symbols cache from %s", cache_file) + log.info("Loading document symbols cache from %s", cache_file) try: - saved_cache = load_cache(str(cache_file), cache_version) + saved_cache = load_cache(str(cache_file), self._raw_document_symbols_cache_version()) if saved_cache is not None: self._raw_document_symbols_cache = saved_cache - self.logger.log(f"Loaded {len(self._raw_document_symbols_cache)} entries from document symbols cache.", logging.INFO) + self.logger.log(f"Loaded {len(self._raw_document_symbols_cache)} entries from raw document symbols cache.", logging.INFO) except Exception as e: # cache can become corrupt, so just skip loading it log.warning( - "Failed to load document symbols cache from %s (%s); Ignoring cache.", + "Failed to load raw document symbols cache from %s (%s); Ignoring cache.", cache_file, e, ) + def _save_document_symbols_cache(self): + cache_file = self._cache_dir / self.DOCUMENT_SYMBOL_CACHE_FILENAME + + if not self._document_symbols_cache_is_modified: + log.debug("No changes to document symbols cache, skipping save") + return + + log.info("Saving updated document symbols cache to %s", cache_file) + try: + save_cache(str(cache_file), self.DOCUMENT_SYMBOL_CACHE_VERSION, self._document_symbols_cache) + self._document_symbols_cache_is_modified = False + except Exception as e: + log.error( + "Failed to save document symbols cache to %s: %s. Note: this may have resulted in a corrupted cache file.", + cache_file, + e, + ) + + def _load_document_symbols_cache(self): + cache_file = self._cache_dir / self.DOCUMENT_SYMBOL_CACHE_FILENAME + if cache_file.exists(): + log.info("Loading document symbols cache from %s", cache_file) + try: + saved_cache = load_cache(str(cache_file), self.DOCUMENT_SYMBOL_CACHE_VERSION) + if saved_cache is not None: + self._document_symbols_cache = saved_cache + self.logger.log(f"Loaded {len(self._document_symbols_cache)} entries from document symbols cache.", logging.INFO) + except Exception as e: + # cache can become corrupt, so just skip loading it + log.warning( + "Failed to load document symbols cache from %s (%s); Ignoring cache.", + cache_file, + e, + ) + + def save_cache(self): + self._save_raw_document_symbols_cache() + self._save_document_symbols_cache() + def request_workspace_symbol(self, query: str) -> list[ls_types.UnifiedSymbolInformation] | None: """ Raise a [workspace/symbol](https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#workspace_symbol) request to the Language Server From 7450fd73f6b0c9b32450b63034b457573fda782a Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 11 Nov 2025 20:48:38 +0100 Subject: [PATCH 15/26] Fix performance issue: Retrieving symbols with include_body=True would re-read the respective file for every single symbol --- src/solidlsp/ls.py | 174 ++++++++++++++++++++++++--------------------- 1 file changed, 91 insertions(+), 83 deletions(-) diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index 90b04e43..a8d99baa 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -954,9 +954,9 @@ class SolidLanguageServer(ABC): where the parent attribute will be the file symbol which in turn may have a package symbol as parent. If you need a symbol tree that contains file symbols as well, you should use `request_full_symbol_tree` instead. """ - # check if the desired result is cached - cache_key = relative_file_path with self.open_file(relative_file_path) as file_data: + # check if the desired result is cached + cache_key = relative_file_path file_hash_and_result = self._document_symbols_cache.get(cache_key) if file_hash_and_result is not None: file_hash, document_symbols = file_hash_and_result @@ -971,89 +971,91 @@ class SolidLanguageServer(ABC): # no cached result: request the root symbols from the language server response = self._request_document_symbols(relative_file_path, file_data) - if response is None: - log.warning( - f"Received None response from the Language Server for document symbols in {relative_file_path}. " - f"This means the language server can't understand this file (possibly due to syntax errors). It may also be due to a bug or misconfiguration of the LS. " - f"Returning empty list", - ) - return DocumentSymbols([]) - - assert isinstance(response, list), f"Unexpected response from Language Server: {response}" - log.debug("Received %d root symbols for %s from the language server", len(response), relative_file_path) - - def turn_item_into_unified_symbol(item: GenericDocumentSymbol) -> ls_types.UnifiedSymbolInformation: - """ - Ensures that the given item is augmented with location, children, and body (if requested), - such that it satisfies the UnifiedSymbolInformation structure. - - :param item: the item to augment - :return: the augmented item (same object as input, but with more attributes and cast to UnifiedSymbolInformation) - """ - item = cast(ls_types.UnifiedSymbolInformation, item) - absolute_path = os.path.join(self.repository_root_path, relative_file_path) - - # handle missing entries in location - if "location" not in item: - uri = pathlib.Path(absolute_path).as_uri() - assert "range" in item - tree_location = ls_types.Location( - uri=uri, - range=item["range"], - absolutePath=absolute_path, - relativePath=relative_file_path, + if response is None: + log.warning( + f"Received None response from the Language Server for document symbols in {relative_file_path}. " + f"This means the language server can't understand this file (possibly due to syntax errors). It may also be due to a bug or misconfiguration of the LS. " + f"Returning empty list", ) - item["location"] = tree_location - location = item["location"] - if "absolutePath" not in location: - location["absolutePath"] = absolute_path - if "relativePath" not in location: - location["relativePath"] = relative_file_path - if "body" not in item: - item["body"] = self.retrieve_symbol_body(item) - # handle missing selectionRange - if "selectionRange" not in item: - if "range" in item: - item["selectionRange"] = item["range"] - else: - item["selectionRange"] = item["location"]["range"] - children = item.get(LSPConstants.CHILDREN, []) - for child in children: - child["parent"] = item - item[LSPConstants.CHILDREN] = children - return item + return DocumentSymbols([]) - flat_all_symbol_list: list[ls_types.UnifiedSymbolInformation] = [] + assert isinstance(response, list), f"Unexpected response from Language Server: {response}" + log.debug("Received %d root symbols for %s from the language server", len(response), relative_file_path) - def convert_nodes_with_common_parent(nodes: list[GenericDocumentSymbol], parent: ls_types.UnifiedSymbolInformation | None) -> None: - """ - Converts the given nodes into UnifiedSymbolInformation with proper parent-child relationships, - adding overload indices for symbols with the same name under the same parent. - """ - total_name_counts = defaultdict(lambda: 0) - for node in nodes: - total_name_counts[node["name"]] += 1 - name_counts = defaultdict(lambda: 0) - for node in nodes: - flat_all_symbol_list.append(node) - usymbol_node = turn_item_into_unified_symbol(node) - if total_name_counts[usymbol_node["name"]] > 1: - usymbol_node["overload_idx"] = name_counts[usymbol_node["name"]] - name_counts[usymbol_node["name"]] += 1 - usymbol_node["parent"] = parent - if usymbol_node["children"]: - convert_nodes_with_common_parent(usymbol_node["children"], usymbol_node) + def turn_item_into_unified_symbol(item: GenericDocumentSymbol) -> ls_types.UnifiedSymbolInformation: + """ + Ensures that the given item is augmented with location, children, and body (if requested), + such that it satisfies the UnifiedSymbolInformation structure. - root_nodes: list[ls_types.UnifiedSymbolInformation] = response - convert_nodes_with_common_parent(root_nodes, None) - document_symbols = DocumentSymbols(root_nodes) + :param item: the item to augment + :return: the augmented item (same object as input, but with more attributes and cast to UnifiedSymbolInformation) + """ + item = cast(ls_types.UnifiedSymbolInformation, item) + absolute_path = os.path.join(self.repository_root_path, relative_file_path) - # update cache - self.logger.log(f"Caching document symbols for {relative_file_path}", logging.DEBUG) - self._document_symbols_cache[cache_key] = (file_data.content_hash, document_symbols) - self._document_symbols_cache_is_modified = True + # handle missing entries in location + if "location" not in item: + uri = pathlib.Path(absolute_path).as_uri() + assert "range" in item + tree_location = ls_types.Location( + uri=uri, + range=item["range"], + absolutePath=absolute_path, + relativePath=relative_file_path, + ) + item["location"] = tree_location + location = item["location"] + if "absolutePath" not in location: + location["absolutePath"] = absolute_path + if "relativePath" not in location: + location["relativePath"] = relative_file_path + if "body" not in item: + item["body"] = self.retrieve_symbol_body(item, file_buffer=file_data) + # handle missing selectionRange + if "selectionRange" not in item: + if "range" in item: + item["selectionRange"] = item["range"] + else: + item["selectionRange"] = item["location"]["range"] + children = item.get(LSPConstants.CHILDREN, []) + for child in children: + child["parent"] = item + item[LSPConstants.CHILDREN] = children + return item - return document_symbols + flat_all_symbol_list: list[ls_types.UnifiedSymbolInformation] = [] + + def convert_nodes_with_common_parent( + nodes: list[GenericDocumentSymbol], parent: ls_types.UnifiedSymbolInformation | None + ) -> None: + """ + Converts the given nodes into UnifiedSymbolInformation with proper parent-child relationships, + adding overload indices for symbols with the same name under the same parent. + """ + total_name_counts = defaultdict(lambda: 0) + for node in nodes: + total_name_counts[node["name"]] += 1 + name_counts = defaultdict(lambda: 0) + for node in nodes: + flat_all_symbol_list.append(node) + usymbol_node = turn_item_into_unified_symbol(node) + if total_name_counts[usymbol_node["name"]] > 1: + usymbol_node["overload_idx"] = name_counts[usymbol_node["name"]] + name_counts[usymbol_node["name"]] += 1 + usymbol_node["parent"] = parent + if usymbol_node["children"]: + convert_nodes_with_common_parent(usymbol_node["children"], usymbol_node) + + root_nodes: list[ls_types.UnifiedSymbolInformation] = response + convert_nodes_with_common_parent(root_nodes, None) + document_symbols = DocumentSymbols(root_nodes) + + # update cache + self.logger.log(f"Caching document symbols for {relative_file_path}", logging.DEBUG) + self._document_symbols_cache[cache_key] = (file_data.content_hash, document_symbols) + self._document_symbols_cache_is_modified = True + + return document_symbols def request_full_symbol_tree( self, within_relative_path: str | None = None, include_body: bool = False @@ -1305,7 +1307,11 @@ class SolidLanguageServer(ABC): return ls_types.Hover(**response) - def retrieve_symbol_body(self, symbol: ls_types.UnifiedSymbolInformation | LSPTypes.DocumentSymbol | LSPTypes.SymbolInformation) -> str: + def retrieve_symbol_body( + self, + symbol: ls_types.UnifiedSymbolInformation | LSPTypes.DocumentSymbol | LSPTypes.SymbolInformation, + file_buffer: LSPFileBuffer | None = None, + ) -> str: """ Load the body of the given symbol. If the body is already contained in the symbol, just return it. """ @@ -1317,9 +1323,11 @@ class SolidLanguageServer(ABC): symbol_start_line = symbol["location"]["range"]["start"]["line"] symbol_end_line = symbol["location"]["range"]["end"]["line"] assert "relativePath" in symbol["location"] - # TODO fixme: duplicate read - symbol_file = self.retrieve_full_file_content(symbol["location"]["relativePath"]) - symbol_lines = symbol_file.split("\n") + if file_buffer is not None: + symbol_file_content = file_buffer.contents + else: + symbol_file_content = self.retrieve_full_file_content(symbol["location"]["relativePath"]) + symbol_lines = symbol_file_content.split("\n") symbol_body = "\n".join(symbol_lines[symbol_start_line : symbol_end_line + 1]) # remove leading indentation From 44d1a404cd2987d2409a05a96ee00f3e64980da2 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 11 Nov 2025 20:51:48 +0100 Subject: [PATCH 16/26] Remove include_body from interface of SolidLS.request_document_symbols (body is always in data structure, callers can decide whether to use it) --- src/serena/cli.py | 6 ++---- src/serena/symbol.py | 2 +- .../language_servers/al_language_server.py | 4 ++-- .../language_servers/bash_language_server.py | 4 ++-- .../language_servers/fortran_language_server.py | 4 ++-- src/solidlsp/language_servers/nixd_ls.py | 4 ++-- src/solidlsp/ls.py | 7 +++---- test/solidlsp/bash/test_bash_basic.py | 16 +++++----------- test/solidlsp/markdown/test_markdown_basic.py | 8 ++++---- test/solidlsp/perl/test_perl_basic.py | 2 +- 10 files changed, 24 insertions(+), 33 deletions(-) diff --git a/src/serena/cli.py b/src/serena/cli.py index 3d73b4c9..3f3d7916 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -478,8 +478,7 @@ class ProjectCommands(AutoRegisteringGroup): files_failed = [] for i, f in enumerate(tqdm(files, desc="Indexing")): try: - ls.request_document_symbols(f, include_body=False) - ls.request_document_symbols(f, include_body=True) + ls.request_document_symbols(f) except Exception as e: log.error(f"Failed to index {f}, continuing.") collected_exceptions.append(e) @@ -538,9 +537,8 @@ class ProjectCommands(AutoRegisteringGroup): try: for ls in ls_mgr.iter_language_servers(): click.echo(f"Indexing for language {ls.language.value} …") - document_symbols = ls.request_document_symbols(file, include_body=False) + document_symbols = ls.request_document_symbols(file) symbols, _ = document_symbols.get_all_symbols_and_roots() - ls.request_document_symbols(file, include_body=True) if verbose: click.echo(f"Symbols in file '{file}':") for symbol in symbols: diff --git a/src/serena/symbol.py b/src/serena/symbol.py index f35294c7..f3dc2270 100644 --- a/src/serena/symbol.py +++ b/src/serena/symbol.py @@ -517,7 +517,7 @@ class LanguageServerSymbolRetriever: if location.relative_path is None: return None lang_server = self.get_language_server(location.relative_path) - document_symbols = lang_server.request_document_symbols(location.relative_path, include_body=False) + document_symbols = lang_server.request_document_symbols(location.relative_path) for symbol_dict in document_symbols.iter_symbols(): symbol = LanguageServerSymbol(symbol_dict) if symbol.location == location: diff --git a/src/solidlsp/language_servers/al_language_server.py b/src/solidlsp/language_servers/al_language_server.py index e2786f10..be83fc5a 100644 --- a/src/solidlsp/language_servers/al_language_server.py +++ b/src/solidlsp/language_servers/al_language_server.py @@ -713,7 +713,7 @@ class ALLanguageServer(SolidLanguageServer): if os.path.isfile(within_abs_path): # Single file case - use parent class implementation - root_nodes = self.request_document_symbols(within_relative_path, include_body=include_body).root_symbols + root_nodes = self.request_document_symbols(within_relative_path).root_symbols return root_nodes # Directory case - scan within this directory @@ -755,7 +755,7 @@ class ALLanguageServer(SolidLanguageServer): try: # Use our overridden request_document_symbols which handles opening self.logger.log(f"AL: Getting symbols for {relative_path}", logging.DEBUG) - all_syms, root_syms = self.request_document_symbols(relative_path, include_body=include_body).get_all_symbols_and_roots() + all_syms, root_syms = self.request_document_symbols(relative_path).get_all_symbols_and_roots() if root_syms: # Create a file-level symbol containing the document symbols diff --git a/src/solidlsp/language_servers/bash_language_server.py b/src/solidlsp/language_servers/bash_language_server.py index 60957425..9942e1f4 100644 --- a/src/solidlsp/language_servers/bash_language_server.py +++ b/src/solidlsp/language_servers/bash_language_server.py @@ -193,7 +193,7 @@ class BashLanguageServer(SolidLanguageServer): else: self.logger.log("Bash server initialization complete", logging.INFO) - def request_document_symbols(self, relative_file_path: str, include_body: bool = False) -> DocumentSymbols: + def request_document_symbols(self, relative_file_path: str) -> DocumentSymbols: """ Request document symbols from bash-language-server via LSP. @@ -215,7 +215,7 @@ class BashLanguageServer(SolidLanguageServer): self.logger.log(f"Requesting document symbols via LSP for {relative_file_path}", logging.DEBUG) # Use the standard LSP approach - bash-language-server handles all function syntaxes correctly - document_symbols = super().request_document_symbols(relative_file_path, include_body) + document_symbols = super().request_document_symbols(relative_file_path) # Log detection results for debugging functions = [s for s in document_symbols.iter_symbols() if s.get("kind") == 12] diff --git a/src/solidlsp/language_servers/fortran_language_server.py b/src/solidlsp/language_servers/fortran_language_server.py index 6858d41d..69ead2d0 100644 --- a/src/solidlsp/language_servers/fortran_language_server.py +++ b/src/solidlsp/language_servers/fortran_language_server.py @@ -138,7 +138,7 @@ class FortranLanguageServer(SolidLanguageServer): return symbol @override - def request_document_symbols(self, relative_file_path: str, include_body: bool = False) -> DocumentSymbols: + def request_document_symbols(self, relative_file_path: str) -> DocumentSymbols: """ Override to fix fortls's incorrect selectionRange bug. @@ -153,7 +153,7 @@ class FortranLanguageServer(SolidLanguageServer): """ # Get symbols from fortls (with incorrect selectionRange) - document_symbols = super().request_document_symbols(relative_file_path, include_body) + document_symbols = super().request_document_symbols(relative_file_path) # Get file content for parsing with self.open_file(relative_file_path) as file_data: diff --git a/src/solidlsp/language_servers/nixd_ls.py b/src/solidlsp/language_servers/nixd_ls.py index 08927052..4af41c7f 100644 --- a/src/solidlsp/language_servers/nixd_ls.py +++ b/src/solidlsp/language_servers/nixd_ls.py @@ -70,7 +70,7 @@ class NixLanguageServer(SolidLanguageServer): return symbol @override - def request_document_symbols(self, relative_file_path: str, include_body: bool = False) -> DocumentSymbols: + def request_document_symbols(self, relative_file_path: str) -> DocumentSymbols: """ Override to extend Nix symbol ranges to include trailing semicolons. @@ -78,7 +78,7 @@ class NixLanguageServer(SolidLanguageServer): statement-level ranges (including semicolons) for proper symbol replacement. """ # Get symbols from parent implementation - document_symbols = super().request_document_symbols(relative_file_path, include_body) + document_symbols = super().request_document_symbols(relative_file_path) # Get file content for range extension file_content = self.language_server.retrieve_full_file_content(relative_file_path) diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index a8d99baa..3f7218ac 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -941,12 +941,11 @@ class SolidLanguageServer(ABC): with self.open_file(relative_file_path) as opened_file_data: return get_symbols(opened_file_data) - def request_document_symbols(self, relative_file_path: str, include_body: bool = False) -> DocumentSymbols: + def request_document_symbols(self, relative_file_path: str) -> DocumentSymbols: """ Retrieves the collection of symbols in the given file :param relative_file_path: The relative path of the file that has the symbols - :param include_body: whether to include the body of the symbols in the result. :return: the collection of symbols in the file. All contained symbols will have a location, children, and a parent attribute, where the parent attribute is None for root symbols. @@ -1090,7 +1089,7 @@ class SolidLanguageServer(ABC): ) return [] else: - root_nodes = self.request_document_symbols(within_relative_path, include_body=include_body).root_symbols + root_nodes = self.request_document_symbols(within_relative_path).root_symbols return root_nodes # Helper function to recursively process directories @@ -1149,7 +1148,7 @@ class SolidLanguageServer(ABC): child["parent"] = package_symbol elif os.path.isfile(contained_dir_or_file_abs_path): - document_symbols = self.request_document_symbols(contained_dir_or_file_rel_path, include_body=include_body) + document_symbols = self.request_document_symbols(contained_dir_or_file_rel_path) file_root_nodes = document_symbols.root_symbols # Create file symbol, link with children diff --git a/test/solidlsp/bash/test_bash_basic.py b/test/solidlsp/bash/test_bash_basic.py index 49bca382..1f2aa995 100644 --- a/test/solidlsp/bash/test_bash_basic.py +++ b/test/solidlsp/bash/test_bash_basic.py @@ -25,7 +25,7 @@ class TestBashLanguageServerBasics: def test_bash_request_document_symbols(self, language_server: SolidLanguageServer) -> None: """Test request_document_symbols for bash files.""" # Test getting symbols from main.sh - all_symbols, _root_symbols = language_server.request_document_symbols("main.sh", include_body=False).get_all_symbols_and_roots() + all_symbols, _root_symbols = language_server.request_document_symbols("main.sh").get_all_symbols_and_roots() # Extract function symbols (LSP Symbol Kind 12) function_symbols = [symbol for symbol in all_symbols if symbol.get("kind") == 12] @@ -41,7 +41,7 @@ class TestBashLanguageServerBasics: def test_bash_request_document_symbols_with_body(self, language_server: SolidLanguageServer) -> None: """Test request_document_symbols with body extraction.""" # Test with include_body=True - all_symbols, _root_symbols = language_server.request_document_symbols("main.sh", include_body=True).get_all_symbols_and_roots() + all_symbols, _root_symbols = language_server.request_document_symbols("main.sh").get_all_symbols_and_roots() function_symbols = [symbol for symbol in all_symbols if symbol.get("kind") == 12] @@ -58,9 +58,7 @@ class TestBashLanguageServerBasics: def test_bash_utils_functions(self, language_server: SolidLanguageServer) -> None: """Test function detection in utils.sh file.""" # Test with utils.sh as well - utils_all_symbols, _utils_root_symbols = language_server.request_document_symbols( - "utils.sh", include_body=False - ).get_all_symbols_and_roots() + utils_all_symbols, _utils_root_symbols = language_server.request_document_symbols("utils.sh").get_all_symbols_and_roots() utils_function_symbols = [symbol for symbol in utils_all_symbols if symbol.get("kind") == 12] utils_function_names = [symbol["name"] for symbol in utils_function_symbols] @@ -86,16 +84,12 @@ class TestBashLanguageServerBasics: def test_bash_function_syntax_patterns(self, language_server: SolidLanguageServer) -> None: """Test that LSP detects different bash function syntax patterns correctly.""" # Test main.sh (has both 'function' keyword and traditional syntax) - main_all_symbols, _main_root_symbols = language_server.request_document_symbols( - "main.sh", include_body=False - ).get_all_symbols_and_roots() + main_all_symbols, _main_root_symbols = language_server.request_document_symbols("main.sh").get_all_symbols_and_roots() main_functions = [symbol for symbol in main_all_symbols if symbol.get("kind") == 12] main_function_names = [func["name"] for func in main_functions] # Test utils.sh (all use 'function' keyword) - utils_all_symbols, _utils_root_symbols = language_server.request_document_symbols( - "utils.sh", include_body=False - ).get_all_symbols_and_roots() + utils_all_symbols, _utils_root_symbols = language_server.request_document_symbols("utils.sh").get_all_symbols_and_roots() utils_functions = [symbol for symbol in utils_all_symbols if symbol.get("kind") == 12] utils_function_names = [func["name"] for func in utils_functions] diff --git a/test/solidlsp/markdown/test_markdown_basic.py b/test/solidlsp/markdown/test_markdown_basic.py index 2c9779da..81a988c0 100644 --- a/test/solidlsp/markdown/test_markdown_basic.py +++ b/test/solidlsp/markdown/test_markdown_basic.py @@ -25,7 +25,7 @@ class TestMarkdownLanguageServerBasics: def test_markdown_request_document_symbols(self, language_server: SolidLanguageServer) -> None: """Test request_document_symbols for markdown files.""" # Test getting symbols from README.md - all_symbols, _root_symbols = language_server.request_document_symbols("README.md", include_body=False).get_all_symbols_and_roots() + all_symbols, _root_symbols = language_server.request_document_symbols("README.md").get_all_symbols_and_roots() # Extract heading symbols (LSP Symbol Kind 15 is String, but marksman uses kind 15 for headings) # Note: Different markdown LSPs may use different symbol kinds for headings @@ -38,7 +38,7 @@ class TestMarkdownLanguageServerBasics: @pytest.mark.parametrize("language_server", [Language.MARKDOWN], indirect=True) def test_markdown_request_symbols_from_guide(self, language_server: SolidLanguageServer) -> None: """Test symbol detection in guide.md file.""" - all_symbols, _root_symbols = language_server.request_document_symbols("guide.md", include_body=False).get_all_symbols_and_roots() + all_symbols, _root_symbols = language_server.request_document_symbols("guide.md").get_all_symbols_and_roots() # At least some headings should be found assert len(all_symbols) > 0, f"Should find headings in guide.md, found {len(all_symbols)}" @@ -46,7 +46,7 @@ class TestMarkdownLanguageServerBasics: @pytest.mark.parametrize("language_server", [Language.MARKDOWN], indirect=True) def test_markdown_request_symbols_from_api(self, language_server: SolidLanguageServer) -> None: """Test symbol detection in api.md file.""" - all_symbols, _root_symbols = language_server.request_document_symbols("api.md", include_body=False).get_all_symbols_and_roots() + all_symbols, _root_symbols = language_server.request_document_symbols("api.md").get_all_symbols_and_roots() # Should detect headings from api.md assert len(all_symbols) > 0, f"Should find headings in api.md, found {len(all_symbols)}" @@ -55,7 +55,7 @@ class TestMarkdownLanguageServerBasics: def test_markdown_request_document_symbols_with_body(self, language_server: SolidLanguageServer) -> None: """Test request_document_symbols with body extraction.""" # Test with include_body=True - all_symbols, _root_symbols = language_server.request_document_symbols("README.md", include_body=True).get_all_symbols_and_roots() + all_symbols, _root_symbols = language_server.request_document_symbols("README.md").get_all_symbols_and_roots() # Should have found some symbols assert len(all_symbols) > 0, "Should find symbols in README.md" diff --git a/test/solidlsp/perl/test_perl_basic.py b/test/solidlsp/perl/test_perl_basic.py index 4c18627f..59dbcc5f 100644 --- a/test/solidlsp/perl/test_perl_basic.py +++ b/test/solidlsp/perl/test_perl_basic.py @@ -31,7 +31,7 @@ class TestPerlLanguageServer: def test_document_symbols(self, language_server: SolidLanguageServer) -> None: """Test that document symbols are correctly identified.""" # Request document symbols - all_symbols, _ = language_server.request_document_symbols("main.pl", include_body=False).get_all_symbols_and_roots() + all_symbols, _ = language_server.request_document_symbols("main.pl").get_all_symbols_and_roots() assert all_symbols, "Expected to find symbols in main.pl" assert len(all_symbols) > 0, "Expected at least one symbol" From c6a5c1aa57eb3df5f98136640214840ff2a9d902 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 11 Nov 2025 22:05:33 +0100 Subject: [PATCH 17/26] Reinstate access to cache location (used by cli) --- src/serena/cli.py | 4 ++-- src/solidlsp/ls.py | 43 ++++++++++++++++++++++--------------------- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/src/serena/cli.py b/src/serena/cli.py index 3f3d7916..9fa88c9c 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -486,7 +486,7 @@ class ProjectCommands(AutoRegisteringGroup): if (i + 1) % 10 == 0: ls.save_cache() ls.save_cache() - click.echo(f"Symbols saved to {ls.cache_path}") + click.echo(f"Symbols saved to {ls.cache_dir}") if len(files_failed) > 0: os.makedirs(os.path.dirname(log_file), exist_ok=True) @@ -544,7 +544,7 @@ class ProjectCommands(AutoRegisteringGroup): for symbol in symbols: click.echo(f" - {symbol['name']} at line {symbol['selectionRange']['start']['line']} of kind {symbol['kind']}") ls.save_cache() - click.echo(f"Successfully indexed file '{file}', {len(symbols)} symbols saved to {ls.cache_path}.") + click.echo(f"Successfully indexed file '{file}', {len(symbols)} symbols saved to cache in {ls.cache_dir}.") finally: ls_mgr.stop_all() diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index 3f7218ac..b62006f7 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -244,10 +244,10 @@ class SolidLanguageServer(ABC): self.language = Language(language_id) # initialise symbol caches - self._cache_dir = ( + self.cache_dir = ( Path(self.repository_root_path) / self._solidlsp_settings.project_data_relative_path / self.CACHE_FOLDER_NAME / self.language_id ) - self._cache_dir.mkdir(parents=True, exist_ok=True) + self.cache_dir.mkdir(parents=True, exist_ok=True) # * raw document symbols cache self._ls_specific_raw_document_symbols_cache_version = cache_version_raw_document_symbols self._raw_document_symbols_cache: dict[str, tuple[str, list[GenericDocumentSymbol]]] = {} @@ -1658,7 +1658,7 @@ class SolidLanguageServer(ABC): return defining_symbol def _save_raw_document_symbols_cache(self): - cache_file = self._cache_dir / self.RAW_DOCUMENT_SYMBOL_CACHE_FILENAME + cache_file = self.cache_dir / self.RAW_DOCUMENT_SYMBOL_CACHE_FILENAME if not self._raw_document_symbols_cache_is_modified: log.debug("No changes to raw document symbols cache, skipping save") @@ -1679,11 +1679,11 @@ class SolidLanguageServer(ABC): return (self.RAW_DOCUMENT_SYMBOLS_CACHE_VERSION, self._ls_specific_raw_document_symbols_cache_version) def _load_raw_document_symbols_cache(self): - cache_file = self._cache_dir / self.RAW_DOCUMENT_SYMBOL_CACHE_FILENAME + cache_file = self.cache_dir / self.RAW_DOCUMENT_SYMBOL_CACHE_FILENAME if not cache_file.exists(): # check for legacy cache to load to migrate - legacy_cache_file = self._cache_dir / self.RAW_DOCUMENT_SYMBOL_CACHE_FILENAME_LEGACY_FALLBACK + legacy_cache_file = self.cache_dir / self.RAW_DOCUMENT_SYMBOL_CACHE_FILENAME_LEGACY_FALLBACK if legacy_cache_file.exists(): try: legacy_cache: dict[ @@ -1704,23 +1704,24 @@ class SolidLanguageServer(ABC): log.error("Failed to migrate cache: %s", e) return - # cache file exists, so load it - log.info("Loading document symbols cache from %s", cache_file) - try: - saved_cache = load_cache(str(cache_file), self._raw_document_symbols_cache_version()) - if saved_cache is not None: - self._raw_document_symbols_cache = saved_cache - self.logger.log(f"Loaded {len(self._raw_document_symbols_cache)} entries from raw document symbols cache.", logging.INFO) - except Exception as e: - # cache can become corrupt, so just skip loading it - log.warning( - "Failed to load raw document symbols cache from %s (%s); Ignoring cache.", - cache_file, - e, - ) + # load existing cache (if any) + if cache_file.exists(): + log.info("Loading document symbols cache from %s", cache_file) + try: + saved_cache = load_cache(str(cache_file), self._raw_document_symbols_cache_version()) + if saved_cache is not None: + self._raw_document_symbols_cache = saved_cache + self.logger.log(f"Loaded {len(self._raw_document_symbols_cache)} entries from raw document symbols cache.", logging.INFO) + except Exception as e: + # cache can become corrupt, so just skip loading it + log.warning( + "Failed to load raw document symbols cache from %s (%s); Ignoring cache.", + cache_file, + e, + ) def _save_document_symbols_cache(self): - cache_file = self._cache_dir / self.DOCUMENT_SYMBOL_CACHE_FILENAME + cache_file = self.cache_dir / self.DOCUMENT_SYMBOL_CACHE_FILENAME if not self._document_symbols_cache_is_modified: log.debug("No changes to document symbols cache, skipping save") @@ -1738,7 +1739,7 @@ class SolidLanguageServer(ABC): ) def _load_document_symbols_cache(self): - cache_file = self._cache_dir / self.DOCUMENT_SYMBOL_CACHE_FILENAME + cache_file = self.cache_dir / self.DOCUMENT_SYMBOL_CACHE_FILENAME if cache_file.exists(): log.info("Loading document symbols cache from %s", cache_file) try: From 10c9cc33f0f873308d12c37272b1669e22fd62cf Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 11 Nov 2025 22:25:47 +0100 Subject: [PATCH 18/26] Remove obsolete list of all symbols --- src/solidlsp/ls.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index b62006f7..a8502084 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -1022,8 +1022,6 @@ class SolidLanguageServer(ABC): item[LSPConstants.CHILDREN] = children return item - flat_all_symbol_list: list[ls_types.UnifiedSymbolInformation] = [] - def convert_nodes_with_common_parent( nodes: list[GenericDocumentSymbol], parent: ls_types.UnifiedSymbolInformation | None ) -> None: @@ -1036,7 +1034,6 @@ class SolidLanguageServer(ABC): total_name_counts[node["name"]] += 1 name_counts = defaultdict(lambda: 0) for node in nodes: - flat_all_symbol_list.append(node) usymbol_node = turn_item_into_unified_symbol(node) if total_name_counts[usymbol_node["name"]] > 1: usymbol_node["overload_idx"] = name_counts[usymbol_node["name"]] From cb26c8c169358bc213ced8737683ea77dd76519b Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 11 Nov 2025 22:53:39 +0100 Subject: [PATCH 19/26] Do not augment existing symbol dictionaries from the LS, instead create new ones because the raw cache would otherwise store the exact same objects --- src/solidlsp/ls.py | 74 +++++++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 34 deletions(-) diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index a8502084..47a183bc 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -968,9 +968,9 @@ class SolidLanguageServer(ABC): log.debug("No cache hit for document symbols in %s", relative_file_path) # no cached result: request the root symbols from the language server - response = self._request_document_symbols(relative_file_path, file_data) + root_symbols = self._request_document_symbols(relative_file_path, file_data) - if response is None: + if root_symbols is None: log.warning( f"Received None response from the Language Server for document symbols in {relative_file_path}. " f"This means the language server can't understand this file (possibly due to syntax errors). It may also be due to a bug or misconfiguration of the LS. " @@ -978,21 +978,22 @@ class SolidLanguageServer(ABC): ) return DocumentSymbols([]) - assert isinstance(response, list), f"Unexpected response from Language Server: {response}" - log.debug("Received %d root symbols for %s from the language server", len(response), relative_file_path) + assert isinstance(root_symbols, list), f"Unexpected response from Language Server: {root_symbols}" + log.debug("Received %d root symbols for %s from the language server", len(root_symbols), relative_file_path) - def turn_item_into_unified_symbol(item: GenericDocumentSymbol) -> ls_types.UnifiedSymbolInformation: + def convert_to_unified_symbol(original_symbol_dict: GenericDocumentSymbol) -> ls_types.UnifiedSymbolInformation: """ - Ensures that the given item is augmented with location, children, and body (if requested), - such that it satisfies the UnifiedSymbolInformation structure. + Converts the given symbol dictionary to the unified representation, ensuring + that all required fields are present (except 'children' which is handled separately). - :param item: the item to augment - :return: the augmented item (same object as input, but with more attributes and cast to UnifiedSymbolInformation) + :param original_symbol_dict: the item to augment + :return: the augmented item (new object) """ - item = cast(ls_types.UnifiedSymbolInformation, item) + # noinspection PyInvalidCast + item = cast(ls_types.UnifiedSymbolInformation, dict(original_symbol_dict)) absolute_path = os.path.join(self.repository_root_path, relative_file_path) - # handle missing entries in location + # handle missing location and path entries if "location" not in item: uri = pathlib.Path(absolute_path).as_uri() assert "range" in item @@ -1008,46 +1009,49 @@ class SolidLanguageServer(ABC): location["absolutePath"] = absolute_path if "relativePath" not in location: location["relativePath"] = relative_file_path + if "body" not in item: item["body"] = self.retrieve_symbol_body(item, file_buffer=file_data) + # handle missing selectionRange if "selectionRange" not in item: if "range" in item: item["selectionRange"] = item["range"] else: item["selectionRange"] = item["location"]["range"] - children = item.get(LSPConstants.CHILDREN, []) - for child in children: - child["parent"] = item - item[LSPConstants.CHILDREN] = children + return item - def convert_nodes_with_common_parent( - nodes: list[GenericDocumentSymbol], parent: ls_types.UnifiedSymbolInformation | None - ) -> None: + def convert_symbols_with_common_parent( + symbols: list[GenericDocumentSymbol], parent: ls_types.UnifiedSymbolInformation | None + ) -> list[ls_types.UnifiedSymbolInformation]: """ - Converts the given nodes into UnifiedSymbolInformation with proper parent-child relationships, + Converts the given symbols into UnifiedSymbolInformation with proper parent-child relationships, adding overload indices for symbols with the same name under the same parent. """ total_name_counts = defaultdict(lambda: 0) - for node in nodes: - total_name_counts[node["name"]] += 1 + for symbol in symbols: + total_name_counts[symbol["name"]] += 1 name_counts = defaultdict(lambda: 0) - for node in nodes: - usymbol_node = turn_item_into_unified_symbol(node) - if total_name_counts[usymbol_node["name"]] > 1: - usymbol_node["overload_idx"] = name_counts[usymbol_node["name"]] - name_counts[usymbol_node["name"]] += 1 - usymbol_node["parent"] = parent - if usymbol_node["children"]: - convert_nodes_with_common_parent(usymbol_node["children"], usymbol_node) + unified_symbols = [] + for symbol in symbols: + usymbol = convert_to_unified_symbol(symbol) + if total_name_counts[usymbol["name"]] > 1: + usymbol["overload_idx"] = name_counts[usymbol["name"]] + name_counts[usymbol["name"]] += 1 + usymbol["parent"] = parent + if "children" in usymbol: + usymbol["children"] = convert_symbols_with_common_parent(usymbol["children"], usymbol) + else: + usymbol["children"] = [] + unified_symbols.append(usymbol) + return unified_symbols - root_nodes: list[ls_types.UnifiedSymbolInformation] = response - convert_nodes_with_common_parent(root_nodes, None) - document_symbols = DocumentSymbols(root_nodes) + unified_root_symbols = convert_symbols_with_common_parent(root_symbols, None) + document_symbols = DocumentSymbols(unified_root_symbols) # update cache - self.logger.log(f"Caching document symbols for {relative_file_path}", logging.DEBUG) + log.debug("Updating cached document symbols for %s", relative_file_path) self._document_symbols_cache[cache_key] = (file_data.content_hash, document_symbols) self._document_symbols_cache_is_modified = True @@ -1708,7 +1712,9 @@ class SolidLanguageServer(ABC): saved_cache = load_cache(str(cache_file), self._raw_document_symbols_cache_version()) if saved_cache is not None: self._raw_document_symbols_cache = saved_cache - self.logger.log(f"Loaded {len(self._raw_document_symbols_cache)} entries from raw document symbols cache.", logging.INFO) + self.logger.log( + f"Loaded {len(self._raw_document_symbols_cache)} entries from raw document symbols cache.", logging.INFO + ) except Exception as e: # cache can become corrupt, so just skip loading it log.warning( From faa0708595adf43b3252c37ab005019698fe148f Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 11 Nov 2025 23:26:53 +0100 Subject: [PATCH 20/26] Improve/optimise logging --- src/solidlsp/ls.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index 47a183bc..0df1487a 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -903,7 +903,7 @@ class SolidLanguageServer(ABC): :return: the list of root symbols in the file. """ - def get_cached_result(cache_key: str, fd: LSPFileBuffer) -> list[GenericDocumentSymbol] | None: + def get_cached_raw_document_symbols(cache_key: str, fd: LSPFileBuffer) -> list[GenericDocumentSymbol] | None: file_hash_and_result = self._raw_document_symbols_cache.get(cache_key) if file_hash_and_result is not None: file_hash, result = file_hash_and_result @@ -911,15 +911,15 @@ class SolidLanguageServer(ABC): log.debug("Returning cached raw document symbols for %s", relative_file_path) return result else: - log.debug("Raw document symbol content for %s has changed. Will overwrite in-memory cache", relative_file_path) + log.debug("Document content for %s has changed (raw symbol cache is not up-to-date)", relative_file_path) else: log.debug("No cache hit for raw document symbols symbols in %s", relative_file_path) return None - def get_symbols(fd: LSPFileBuffer) -> list[GenericDocumentSymbol]: + def get_raw_document_symbols(fd: LSPFileBuffer) -> list[GenericDocumentSymbol]: # check for cached result cache_key = relative_file_path - response = get_cached_result(cache_key, fd) + response = get_cached_raw_document_symbols(cache_key, fd) if response is not None: return response @@ -936,10 +936,10 @@ class SolidLanguageServer(ABC): return response if file_data is not None: - return get_symbols(file_data) + return get_raw_document_symbols(file_data) else: with self.open_file(relative_file_path) as opened_file_data: - return get_symbols(opened_file_data) + return get_raw_document_symbols(opened_file_data) def request_document_symbols(self, relative_file_path: str) -> DocumentSymbols: """ @@ -1084,10 +1084,7 @@ class SolidLanguageServer(ABC): raise FileNotFoundError(f"File or directory not found: {within_abs_path}") if os.path.isfile(within_abs_path): if self.is_ignored_path(within_relative_path): - self.logger.log( - f"You passed a file explicitly, but it is ignored. This is probably an error. File: {within_relative_path}", - logging.ERROR, - ) + log.error("You passed a file explicitly, but it is ignored. This is probably an error. File: %s", within_relative_path) return [] else: root_nodes = self.request_document_symbols(within_relative_path).root_symbols @@ -1099,7 +1096,7 @@ class SolidLanguageServer(ABC): abs_dir_path = os.path.realpath(abs_dir_path) if self.is_ignored_path(str(Path(abs_dir_path).relative_to(self.repository_root_path))): - self.logger.log(f"Skipping directory: {rel_dir_path}\n(because it should be ignored)", logging.DEBUG) + log.debug("Skipping directory: %s (because it should be ignored)", rel_dir_path) return [] result = [] @@ -1132,14 +1129,16 @@ class SolidLanguageServer(ABC): ) except ValueError as e: # Typically happens when the path is not under the repository root (e.g., symlink pointing outside) - self.logger.log( - f"Skipping path {contained_dir_or_file_abs_path}; likely outside of the repository root {self.repository_root_path} [cause: {e}]", - logging.WARNING, + log.warning( + "Skipping path %s; likely outside of the repository root %s [cause: %s]", + contained_dir_or_file_abs_path, + self.repository_root_path, + e, ) continue if self.is_ignored_path(contained_dir_or_file_rel_path): - self.logger.log(f"Skipping item: {contained_dir_or_file_rel_path}\n(because it should be ignored)", logging.DEBUG) + log.debug("Skipping item: %s (because it should be ignored)", contained_dir_or_file_rel_path) continue if os.path.isdir(contained_dir_or_file_abs_path): From 44c1b69ddee9969583655e9abc5268ace3bd4abd Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 12 Nov 2025 03:23:18 +0100 Subject: [PATCH 21/26] Open every file exactly once during request_full_symbol_tree --- .../language_servers/bash_language_server.py | 29 +++------ .../fortran_language_server.py | 28 ++++----- src/solidlsp/language_servers/nixd_ls.py | 14 ++--- src/solidlsp/ls.py | 62 ++++++++++++------- 4 files changed, 66 insertions(+), 67 deletions(-) diff --git a/src/solidlsp/language_servers/bash_language_server.py b/src/solidlsp/language_servers/bash_language_server.py index 9942e1f4..f7dd1d5e 100644 --- a/src/solidlsp/language_servers/bash_language_server.py +++ b/src/solidlsp/language_servers/bash_language_server.py @@ -10,7 +10,7 @@ import shutil import threading from solidlsp.language_servers.common import RuntimeDependency, RuntimeDependencyCollection -from solidlsp.ls import DocumentSymbols, SolidLanguageServer +from solidlsp.ls import DocumentSymbols, LSPFileBuffer, SolidLanguageServer from solidlsp.ls_config import LanguageServerConfig from solidlsp.ls_logger import LanguageServerLogger from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams @@ -193,29 +193,18 @@ class BashLanguageServer(SolidLanguageServer): else: self.logger.log("Bash server initialization complete", logging.INFO) - def request_document_symbols(self, relative_file_path: str) -> DocumentSymbols: - """ - Request document symbols from bash-language-server via LSP. + def request_document_symbols(self, relative_file_path: str, file_buffer: LSPFileBuffer | None = None) -> DocumentSymbols: + # Uses the standard LSP documentSymbol request which provides reliable function detection + # for all bash function syntaxes including: + # - function name() { ... } (with function keyword) + # - name() { ... } (traditional syntax) + # - Functions with various indentation levels + # - Functions with comments before/after/inside - Uses the standard LSP documentSymbol request which provides reliable function detection - for all bash function syntaxes including: - - function name() { ... } (with function keyword) - - name() { ... } (traditional syntax) - - Functions with various indentation levels - - Functions with comments before/after/inside - - Args: - relative_file_path: Path to the bash file relative to repository root - include_body: Whether to include function bodies in symbol information - - Returns: - The symbols - - """ self.logger.log(f"Requesting document symbols via LSP for {relative_file_path}", logging.DEBUG) # Use the standard LSP approach - bash-language-server handles all function syntaxes correctly - document_symbols = super().request_document_symbols(relative_file_path) + document_symbols = super().request_document_symbols(relative_file_path, file_buffer=file_buffer) # Log detection results for debugging functions = [s for s in document_symbols.iter_symbols() if s.get("kind") == 12] diff --git a/src/solidlsp/language_servers/fortran_language_server.py b/src/solidlsp/language_servers/fortran_language_server.py index 69ead2d0..782df7b3 100644 --- a/src/solidlsp/language_servers/fortran_language_server.py +++ b/src/solidlsp/language_servers/fortran_language_server.py @@ -12,7 +12,7 @@ import threading from overrides import override from solidlsp import ls_types -from solidlsp.ls import DocumentSymbols, SolidLanguageServer +from solidlsp.ls import DocumentSymbols, LSPFileBuffer, SolidLanguageServer from solidlsp.ls_config import LanguageServerConfig from solidlsp.ls_logger import LanguageServerLogger from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams @@ -138,22 +138,20 @@ class FortranLanguageServer(SolidLanguageServer): return symbol @override - def request_document_symbols(self, relative_file_path: str) -> DocumentSymbols: - """ - Override to fix fortls's incorrect selectionRange bug. + def request_document_symbols(self, relative_file_path: str, file_buffer: LSPFileBuffer | None = None) -> DocumentSymbols: + # Override to fix fortls's incorrect selectionRange bug. + # + # fortls returns selectionRange pointing to line start (character 0) instead of the + # identifier name position. This breaks MCP server features that rely on exact positions. + # + # This override: + # 1. Gets symbols from fortls via parent implementation + # 2. Parses each symbol's line to find the correct identifier position + # 3. Fixes selectionRange for all symbols recursively + # 4. Returns corrected symbols - fortls returns selectionRange pointing to line start (character 0) instead of the - identifier name position. This breaks MCP server features that rely on exact positions. - - This override: - 1. Gets symbols from fortls via parent implementation - 2. Parses each symbol's line to find the correct identifier position - 3. Fixes selectionRange for all symbols recursively - 4. Returns corrected symbols - - """ # Get symbols from fortls (with incorrect selectionRange) - document_symbols = super().request_document_symbols(relative_file_path) + document_symbols = super().request_document_symbols(relative_file_path, file_buffer=file_buffer) # Get file content for parsing with self.open_file(relative_file_path) as file_data: diff --git a/src/solidlsp/language_servers/nixd_ls.py b/src/solidlsp/language_servers/nixd_ls.py index 4af41c7f..beea7c6b 100644 --- a/src/solidlsp/language_servers/nixd_ls.py +++ b/src/solidlsp/language_servers/nixd_ls.py @@ -16,7 +16,7 @@ from pathlib import Path from overrides import override from solidlsp import ls_types -from solidlsp.ls import DocumentSymbols, SolidLanguageServer +from solidlsp.ls import DocumentSymbols, LSPFileBuffer, SolidLanguageServer from solidlsp.ls_config import LanguageServerConfig from solidlsp.ls_logger import LanguageServerLogger from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams @@ -70,15 +70,13 @@ class NixLanguageServer(SolidLanguageServer): return symbol @override - def request_document_symbols(self, relative_file_path: str) -> DocumentSymbols: - """ - Override to extend Nix symbol ranges to include trailing semicolons. + def request_document_symbols(self, relative_file_path: str, file_buffer: LSPFileBuffer | None = None) -> DocumentSymbols: + # Override to extend Nix symbol ranges to include trailing semicolons. + # nixd provides expression-level ranges (excluding semicolons) but serena needs + # statement-level ranges (including semicolons) for proper symbol replacement. - nixd provides expression-level ranges (excluding semicolons) but serena needs - statement-level ranges (including semicolons) for proper symbol replacement. - """ # Get symbols from parent implementation - document_symbols = super().request_document_symbols(relative_file_path) + document_symbols = super().request_document_symbols(relative_file_path, file_buffer=file_buffer) # Get file content for range extension file_content = self.language_server.retrieve_full_file_content(relative_file_path) diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index 0df1487a..2341abba 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -484,6 +484,20 @@ class SolidLanguageServer(ABC): ) del self.open_file_buffers[uri] + @contextmanager + def _open_file_context(self, relative_file_path: str, file_buffer: LSPFileBuffer | None = None) -> Iterator[LSPFileBuffer]: + """ + Internal context manager to open a file, optionally reusing an existing file buffer. + + :param relative_file_path: the relative path of the file to open. + :param file_buffer: an optional existing file buffer to reuse. + """ + if file_buffer is not None: + yield file_buffer + else: + with self.open_file(relative_file_path) as fb: + yield fb + def insert_text_at_position(self, relative_file_path: str, line: int, column: int, text_to_be_inserted: str) -> ls_types.Position: """ Insert text at the given line and column in the given file and return @@ -941,11 +955,12 @@ class SolidLanguageServer(ABC): with self.open_file(relative_file_path) as opened_file_data: return get_raw_document_symbols(opened_file_data) - def request_document_symbols(self, relative_file_path: str) -> DocumentSymbols: + def request_document_symbols(self, relative_file_path: str, file_buffer: LSPFileBuffer | None = None) -> DocumentSymbols: """ Retrieves the collection of symbols in the given file :param relative_file_path: The relative path of the file that has the symbols + :param file_buffer: an optional file buffer if the file is already opened. :return: the collection of symbols in the file. All contained symbols will have a location, children, and a parent attribute, where the parent attribute is None for root symbols. @@ -953,7 +968,7 @@ class SolidLanguageServer(ABC): where the parent attribute will be the file symbol which in turn may have a package symbol as parent. If you need a symbol tree that contains file symbols as well, you should use `request_full_symbol_tree` instead. """ - with self.open_file(relative_file_path) as file_data: + with self._open_file_context(relative_file_path, file_buffer) as file_data: # check if the desired result is cached cache_key = relative_file_path file_hash_and_result = self._document_symbols_cache.get(cache_key) @@ -1148,29 +1163,28 @@ class SolidLanguageServer(ABC): child["parent"] = package_symbol elif os.path.isfile(contained_dir_or_file_abs_path): - document_symbols = self.request_document_symbols(contained_dir_or_file_rel_path) - file_root_nodes = document_symbols.root_symbols + with self._open_file_context(contained_dir_or_file_rel_path) as file_data: + document_symbols = self.request_document_symbols(contained_dir_or_file_rel_path, file_data) + file_root_nodes = document_symbols.root_symbols - # Create file symbol, link with children - file_rel_path = str(Path(contained_dir_or_file_abs_path).resolve().relative_to(self.repository_root_path)) - with self.open_file(file_rel_path) as file_data: - fileRange = self._get_range_from_file_content(file_data.contents) - file_symbol = ls_types.UnifiedSymbolInformation( # type: ignore - name=os.path.splitext(contained_dir_or_file_name)[0], - kind=ls_types.SymbolKind.File, - range=fileRange, - selectionRange=fileRange, - location=ls_types.Location( - uri=str(pathlib.Path(contained_dir_or_file_abs_path).as_uri()), - range=fileRange, - absolutePath=str(contained_dir_or_file_abs_path), - relativePath=str(Path(contained_dir_or_file_abs_path).resolve().relative_to(self.repository_root_path)), - ), - children=file_root_nodes, - parent=package_symbol, - ) - for child in file_root_nodes: - child["parent"] = file_symbol + # Create file symbol, link with children + file_range = self._get_range_from_file_content(file_data.contents) + file_symbol = ls_types.UnifiedSymbolInformation( # type: ignore + name=os.path.splitext(contained_dir_or_file_name)[0], + kind=ls_types.SymbolKind.File, + range=file_range, + selectionRange=file_range, + location=ls_types.Location( + uri=str(pathlib.Path(contained_dir_or_file_abs_path).as_uri()), + range=file_range, + absolutePath=str(contained_dir_or_file_abs_path), + relativePath=str(Path(contained_dir_or_file_abs_path).resolve().relative_to(self.repository_root_path)), + ), + children=file_root_nodes, + parent=package_symbol, + ) + for child in file_root_nodes: + child["parent"] = file_symbol # Link file symbol with package package_symbol["children"].append(file_symbol) From 0699a8726452411beba70f103613088e7f9d95cd Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 12 Nov 2025 03:32:40 +0100 Subject: [PATCH 22/26] request_document_symbols: Avoid splitting document into lines more than once (optimisation) --- src/solidlsp/ls.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index 2341abba..91ac8966 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -79,6 +79,10 @@ class LSPFileBuffer: def __post_init__(self): self.content_hash = hashlib.md5(self.contents.encode("utf-8")).hexdigest() + def split_lines(self) -> list[str]: + """Splits the contents of the file into lines.""" + return self.contents.split("\n") + class DocumentSymbols: # IMPORTANT: Instances of this class are persisted in the high-level document symbol cache @@ -996,6 +1000,8 @@ class SolidLanguageServer(ABC): assert isinstance(root_symbols, list), f"Unexpected response from Language Server: {root_symbols}" log.debug("Received %d root symbols for %s from the language server", len(root_symbols), relative_file_path) + file_lines = file_data.split_lines() + def convert_to_unified_symbol(original_symbol_dict: GenericDocumentSymbol) -> ls_types.UnifiedSymbolInformation: """ Converts the given symbol dictionary to the unified representation, ensuring @@ -1026,7 +1032,7 @@ class SolidLanguageServer(ABC): location["relativePath"] = relative_file_path if "body" not in item: - item["body"] = self.retrieve_symbol_body(item, file_buffer=file_data) + item["body"] = self.retrieve_symbol_body(item, file_lines=file_lines) # handle missing selectionRange if "selectionRange" not in item: @@ -1323,6 +1329,7 @@ class SolidLanguageServer(ABC): def retrieve_symbol_body( self, symbol: ls_types.UnifiedSymbolInformation | LSPTypes.DocumentSymbol | LSPTypes.SymbolInformation, + file_lines: list[str] | None = None, file_buffer: LSPFileBuffer | None = None, ) -> str: """ @@ -1336,12 +1343,10 @@ class SolidLanguageServer(ABC): symbol_start_line = symbol["location"]["range"]["start"]["line"] symbol_end_line = symbol["location"]["range"]["end"]["line"] assert "relativePath" in symbol["location"] - if file_buffer is not None: - symbol_file_content = file_buffer.contents - else: - symbol_file_content = self.retrieve_full_file_content(symbol["location"]["relativePath"]) - symbol_lines = symbol_file_content.split("\n") - symbol_body = "\n".join(symbol_lines[symbol_start_line : symbol_end_line + 1]) + if file_lines is None: + with self._open_file_context(symbol["location"]["relativePath"], file_buffer) as f: + file_lines = f.split_lines() + symbol_body = "\n".join(file_lines[symbol_start_line : symbol_end_line + 1]) # remove leading indentation symbol_start_column = symbol["location"]["range"]["start"]["character"] From 5c7ad8edf7d8cb43956cdbb4c2b22f315e8c99bc Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Thu, 13 Nov 2025 16:37:08 +0700 Subject: [PATCH 23/26] Minor [ci skip] --- scripts/profile_tool_call.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/profile_tool_call.py b/scripts/profile_tool_call.py index 8fe922a5..e0e69981 100644 --- a/scripts/profile_tool_call.py +++ b/scripts/profile_tool_call.py @@ -25,6 +25,7 @@ if __name__ == "__main__": serena_config = SerenaConfig.from_config_file() serena_config.log_level = logging.INFO + serena_config.jetbrains = False serena_config.gui_log_window_enabled = False serena_config.web_dashboard = False From c6e43c3cf369ca4e7849cbcf34798ac75066d20c Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 13 Nov 2025 11:46:53 +0100 Subject: [PATCH 24/26] Improve docstring --- src/solidlsp/ls.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index 91ac8966..c64c05bd 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -133,7 +133,10 @@ class SolidLanguageServer(ABC): CACHE_FOLDER_NAME = "cache" RAW_DOCUMENT_SYMBOLS_CACHE_VERSION = 1 """ - global version identifier for raw symbol caches; an LS-specific version is defined separately and combined with this + global version identifier for raw symbol caches; an LS-specific version is defined separately and combined with this. + This should be incremented whenever there is a change in the way raw document symbols are stored. + If the result of a language server changes in a way that affects the raw document symbols, + the LS-specific version should be incremented instead. """ RAW_DOCUMENT_SYMBOL_CACHE_FILENAME = "raw_document_symbols.pkl" RAW_DOCUMENT_SYMBOL_CACHE_FILENAME_LEGACY_FALLBACK = "document_symbols_cache_v23-06-25.pkl" From 058803a60169a33edda34c0079de967f2617efde Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 13 Nov 2025 13:25:52 +0100 Subject: [PATCH 25/26] Remove obsolete parameter include_body --- src/serena/symbol.py | 3 +-- src/serena/tools/symbol_tools.py | 1 - src/solidlsp/language_servers/al_language_server.py | 2 +- src/solidlsp/ls.py | 6 +----- 4 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/serena/symbol.py b/src/serena/symbol.py index f3dc2270..e3753041 100644 --- a/src/serena/symbol.py +++ b/src/serena/symbol.py @@ -491,7 +491,6 @@ class LanguageServerSymbolRetriever: def find_by_name( self, name_path: str, - include_body: bool = False, include_kinds: Sequence[SymbolKind] | None = None, exclude_kinds: Sequence[SymbolKind] | None = None, substring_matching: bool = False, @@ -504,7 +503,7 @@ class LanguageServerSymbolRetriever: """ symbols: list[LanguageServerSymbol] = [] for lang_server in self._ls_manager.iter_language_servers(): - symbol_roots = lang_server.request_full_symbol_tree(within_relative_path=within_relative_path, include_body=include_body) + symbol_roots = lang_server.request_full_symbol_tree(within_relative_path=within_relative_path) for root in symbol_roots: symbols.extend( LanguageServerSymbol(root).find( diff --git a/src/serena/tools/symbol_tools.py b/src/serena/tools/symbol_tools.py index bd31fc48..01c0dfaa 100644 --- a/src/serena/tools/symbol_tools.py +++ b/src/serena/tools/symbol_tools.py @@ -150,7 +150,6 @@ class FindSymbolTool(Tool, ToolMarkerSymbolicRead): symbol_retriever = self.create_language_server_symbol_retriever() symbols = symbol_retriever.find_by_name( name_path, - include_body=include_body, include_kinds=parsed_include_kinds, exclude_kinds=parsed_exclude_kinds, substring_matching=substring_matching, diff --git a/src/solidlsp/language_servers/al_language_server.py b/src/solidlsp/language_servers/al_language_server.py index be83fc5a..2bc1b1ad 100644 --- a/src/solidlsp/language_servers/al_language_server.py +++ b/src/solidlsp/language_servers/al_language_server.py @@ -680,7 +680,7 @@ class ALLanguageServer(SolidLanguageServer): return super().is_ignored_dirname(dirname) or dirname in al_ignore_dirs @override - def request_full_symbol_tree(self, within_relative_path: str | None = None, include_body: bool = False) -> list[dict]: + def request_full_symbol_tree(self, within_relative_path: str | None = None) -> list[dict]: """ Override to handle AL's requirement of opening files before requesting symbols. diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index c64c05bd..ce91fb45 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -1081,9 +1081,7 @@ class SolidLanguageServer(ABC): return document_symbols - def request_full_symbol_tree( - self, within_relative_path: str | None = None, include_body: bool = False - ) -> list[ls_types.UnifiedSymbolInformation]: + def request_full_symbol_tree(self, within_relative_path: str | None = None) -> list[ls_types.UnifiedSymbolInformation]: """ Will go through all files in the project or within a relative path and build a tree of symbols. Note: this may be slow the first time it is called, especially if `within_relative_path` is not used to restrict the search. @@ -1098,8 +1096,6 @@ class SolidLanguageServer(ABC): :param within_relative_path: pass a relative path to only consider symbols within this path. If a file is passed, only the symbols within this file will be considered. If a directory is passed, all files within this directory will be considered. - :param include_body: whether to include the body of the symbols in the result. - :return: A list of root symbols representing the top-level packages/modules in the project. """ if within_relative_path is not None: From 70e64de890bd1998970e51b86438b5f53b571f3c Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 17 Nov 2025 16:08:39 +0100 Subject: [PATCH 26/26] Save migrated cache and delete legacy cache after migration --- src/solidlsp/ls.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index ce91fb45..06ba0285 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -1717,9 +1717,12 @@ class SolidLanguageServer(ABC): num_symbols_migrated += len(all_symbols) log.info("Migrated %d document symbols from legacy cache", num_symbols_migrated) self._raw_document_symbols_cache = migrated_cache + self._raw_document_symbols_cache_is_modified = True + self._save_raw_document_symbols_cache() + legacy_cache_file.unlink() return except Exception as e: - log.error("Failed to migrate cache: %s", e) + log.error("Error during cache migration: %s", e) return # load existing cache (if any)