diff --git a/src/serena/cli.py b/src/serena/cli.py index 33595e70..711e8e9a 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -39,6 +39,7 @@ from serena.tools import FindReferencingSymbolsTool, FindSymbolTool, GetSymbolsO from serena.util.dataclass import get_dataclass_default from serena.util.logging import MemoryLogHandler from solidlsp.ls_config import Language +from solidlsp.ls_types import SymbolKind from solidlsp.util.subprocess_util import subprocess_kwargs log = logging.getLogger(__name__) @@ -811,9 +812,7 @@ class ProjectCommands(AutoRegisteringGroup): return # Extract suitable symbol (prefer class or function over variables) - # LSP symbol kinds: 5=class, 12=function, 6=method, 9=constructor - preferred_kinds = [5, 12, 6, 9] # class, function, method, constructor - + preferred_kinds = {SymbolKind.Class.name, SymbolKind.Function.name, SymbolKind.Method.name, SymbolKind.Constructor.name} selected_symbol = None for symbol in overview_data: if symbol.get("kind") in preferred_kinds: diff --git a/src/serena/code_editor.py b/src/serena/code_editor.py index 2b848c72..b5075c38 100644 --- a/src/serena/code_editor.py +++ b/src/serena/code_editor.py @@ -6,6 +6,7 @@ from collections.abc import Iterable, Iterator, Reversible from contextlib import contextmanager from typing import TYPE_CHECKING, Generic, Optional, TypeVar, cast +from serena.jetbrains.jetbrains_plugin_client import JetBrainsPluginClient from serena.symbol import JetBrainsSymbol, LanguageServerSymbol, LanguageServerSymbolRetriever, PositionInFile, Symbol from solidlsp import SolidLanguageServer, ls_types from solidlsp.ls import LSPFileBuffer @@ -13,7 +14,6 @@ from solidlsp.ls_utils import PathUtils, TextUtils from .constants import DEFAULT_SOURCE_FILE_ENCODING from .project import Project -from .tools.jetbrains_plugin_client import JetBrainsPluginClient if TYPE_CHECKING: from .agent import SerenaAgent diff --git a/src/serena/tools/jetbrains_plugin_client.py b/src/serena/jetbrains/jetbrains_plugin_client.py similarity index 99% rename from src/serena/tools/jetbrains_plugin_client.py rename to src/serena/jetbrains/jetbrains_plugin_client.py index 5474f051..2607f547 100644 --- a/src/serena/tools/jetbrains_plugin_client.py +++ b/src/serena/jetbrains/jetbrains_plugin_client.py @@ -12,10 +12,10 @@ import requests from requests import Response from sensai.util.string import ToStringMixin -import serena.tools.jetbrains_types as jb +import serena.jetbrains.jetbrains_types as jb +from serena.jetbrains.jetbrains_types import PluginStatusDTO from serena.project import Project from serena.text_utils import render_html -from serena.tools.jetbrains_types import PluginStatusDTO from serena.util.version import Version T = TypeVar("T") diff --git a/src/serena/tools/jetbrains_types.py b/src/serena/jetbrains/jetbrains_types.py similarity index 86% rename from src/serena/tools/jetbrains_types.py rename to src/serena/jetbrains/jetbrains_types.py index 80f02947..fd72bc60 100644 --- a/src/serena/tools/jetbrains_types.py +++ b/src/serena/jetbrains/jetbrains_types.py @@ -1,4 +1,4 @@ -from typing import NotRequired, TypedDict +from typing import Literal, NotRequired, TypedDict class PluginStatusDTO(TypedDict): @@ -30,6 +30,9 @@ class SymbolDTO(TypedDict): num_usages: NotRequired[int] +SymbolDTOKey = Literal["name_path", "relative_path", "type", "body", "quick_info", "documentation", "text_range", "children", "num_usages"] + + class SymbolCollectionResponse(TypedDict): symbols: list[SymbolDTO] diff --git a/src/serena/symbol.py b/src/serena/symbol.py index e91856c1..dc02f58c 100644 --- a/src/serena/symbol.py +++ b/src/serena/symbol.py @@ -4,11 +4,11 @@ import os from abc import ABC, abstractmethod from collections.abc import Callable, Iterator, Sequence from dataclasses import asdict, dataclass -from typing import TYPE_CHECKING, Any, Self, Union +from typing import TYPE_CHECKING, Any, Generic, Literal, NotRequired, Self, TypedDict, TypeVar, Union from sensai.util.string import ToStringMixin -import serena.tools.jetbrains_types as jb +import serena.jetbrains.jetbrains_types as jb from solidlsp import SolidLanguageServer from solidlsp.ls import ReferenceInSymbol as LSPReferenceInSymbol from solidlsp.ls_types import Position, SymbolKind, UnifiedSymbolInformation @@ -212,14 +212,17 @@ class LanguageServerSymbol(Symbol, ToStringMixin): return [] def _tostring_additional_entries(self) -> dict[str, Any]: - return dict(name=self.name, kind=self.kind, num_children=len(self.symbol_root["children"])) + return dict(name=self.name, kind=self.symbol_kind_name, num_children=len(self.symbol_root["children"])) @property def name(self) -> str: return self.symbol_root["name"] @property - def kind(self) -> str: + def symbol_kind_name(self) -> str: + """ + :return: string representation of the symbol kind (name attribute of the `SymbolKind` enum item) + """ return SymbolKind(self.symbol_kind).name @property @@ -395,67 +398,99 @@ class LanguageServerSymbol(Symbol, ToStringMixin): traverse(self) return result + class OutputDict(TypedDict): + name_path: NotRequired[str] + name: NotRequired[str] + location: NotRequired[dict[str, Any]] + relative_path: NotRequired[str | None] + body_location: NotRequired[dict[str, Any]] + body: NotRequired[str | None] + kind: NotRequired[str] + """ + string representation of the symbol kind (name attribute of the `SymbolKind` enum item) + """ + children: NotRequired[list["LanguageServerSymbol.OutputDict"]] + + OutputDictKey = Literal["name", "name_path", "relative_path", "location", "body_location", "body", "kind", "children"] + def to_dict( self, + *, + name_path: bool = True, + name: bool = False, kind: bool = False, location: bool = False, depth: int = 0, - include_body: bool = False, - include_children_body: bool = False, - include_relative_path: bool = True, + body: bool = False, + body_location: bool = False, + children_body: bool = False, + relative_path: bool = False, child_inclusion_predicate: Callable[[Self], bool] | None = None, - ) -> dict[str, Any]: + ) -> OutputDict: """ Converts the symbol to a dictionary. + :param name_path: whether to include the name path of the symbol + :param name: whether to include the name of the symbol :param kind: whether to include the kind of the symbol :param location: whether to include the location of the symbol :param depth: the depth up to which to include child symbols (0 = do not include children) - :param include_body: whether to include the body of the top-level symbol. - :param include_children_body: whether to also include the body of the children. + :param body: whether to include the body of the top-level symbol. + :param children_body: whether to also include the body of the children. Note that the body of the children is part of the body of the parent symbol, so there is usually no need to set this to True unless you want process the output and pass the children without passing the parent body to the LM. - :param include_relative_path: whether to include the relative path of the symbol in the location - entry. Relative paths of the symbol's children are always excluded. + :param relative_path: whether to include the relative path of the symbol. + If `location` is True, this defines whether to include the path in the location entry. + If `location` is False, this defines whether to include the relative path as a top-level entry. + Relative paths of the symbol's children are always excluded. :param child_inclusion_predicate: an optional predicate that decides whether a child symbol should be included. :return: a dictionary representation of the symbol """ - result: dict[str, Any] = {"name": self.name, "name_path": self.get_name_path()} + result: LanguageServerSymbol.OutputDict = {} + + if name_path: + result["name_path"] = self.get_name_path() + if name: + result["name"] = self.name if kind: - result["kind"] = self.kind + result["kind"] = self.symbol_kind_name if location: - result["location"] = self.location.to_dict(include_relative_path=include_relative_path) + result["location"] = self.location.to_dict(include_relative_path=relative_path) + elif relative_path: + result["relative_path"] = self.relative_path + + if body_location: body_start_line, body_end_line = self.get_body_line_numbers() result["body_location"] = {"start_line": body_start_line, "end_line": body_end_line} - if include_body: - body = self.body - if body is None: - log.warning("Requested body for symbol, but it is not present. The symbol might have been loaded with include_body=False.") - result["body"] = body + if body: + result["body"] = self.body if child_inclusion_predicate is None: child_inclusion_predicate = lambda s: True - def included_children(s: Self) -> list[dict[str, Any]]: + def included_children(s: Self) -> list[LanguageServerSymbol.OutputDict]: children = [] for c in s.iter_children(): if not child_inclusion_predicate(c): continue children.append( c.to_dict( + name_path=name_path, + name=name, kind=kind, location=location, + body_location=body_location, depth=depth - 1, child_inclusion_predicate=child_inclusion_predicate, - include_body=include_children_body, - include_children_body=include_children_body, + body=children_body, + children_body=children_body, # all children have the same relative path as the parent - include_relative_path=False, + relative_path=False, ) ) return children @@ -463,7 +498,7 @@ class LanguageServerSymbol(Symbol, ToStringMixin): if depth > 0: children = included_children(self) if len(children) > 0: - result["children"] = included_children(self) + result["children"] = children return result @@ -604,8 +639,7 @@ class LanguageServerSymbolRetriever: include_rel_path = within_relative_path is not None raise ValueError( f"Found multiple {len(symbol_candidates)} symbols matching '{name_path_pattern}'. " - "They are: \n" - + json.dumps([s.to_dict(kind=True, include_relative_path=include_rel_path) for s in symbol_candidates], indent=2) + "They are: \n" + json.dumps([s.to_dict(kind=True, relative_path=include_rel_path) for s in symbol_candidates], indent=2) ) def find_by_location(self, location: LanguageServerSymbolLocation) -> LanguageServerSymbol | None: @@ -689,36 +723,15 @@ class LanguageServerSymbolRetriever: return [ReferenceInLanguageServerSymbol.from_lsp_reference(r) for r in references] - def get_symbol_overview(self, relative_path: str, depth: int = 0) -> dict[str, list[dict]]: + def get_symbol_overview(self, relative_path: str) -> dict[str, list[LanguageServerSymbol]]: """ :param relative_path: the path of the file or directory for which to get the symbol overview - :param depth: the depth up to which to include child symbols (0 = only top-level symbols) - :return: a mapping from file paths to lists of symbol dictionaries. + :return: a mapping from file paths to lists of symbols. For the case where a file is passed, the mapping will contain a single entry. """ lang_server = self.get_language_server(relative_path) path_to_unified_symbols = lang_server.request_overview(relative_path) - - def child_inclusion_predicate(s: LanguageServerSymbol) -> bool: - return not s.is_low_level() - - result = {} - for file_path, unified_symbols in path_to_unified_symbols.items(): - symbols_in_file = [] - for us in unified_symbols: - symbol = LanguageServerSymbol(us) - symbols_in_file.append( - symbol.to_dict( - depth=depth, - kind=True, - include_relative_path=False, - location=False, - child_inclusion_predicate=child_inclusion_predicate, - ) - ) - result[file_path] = symbols_in_file - - return result + return {k: [LanguageServerSymbol(us) for us in v] for k, v in path_to_unified_symbols.items()} class JetBrainsSymbol(Symbol): @@ -775,3 +788,129 @@ class JetBrainsSymbol(Symbol): def is_neighbouring_definition_separated_by_empty_line(self) -> bool: # NOTE: Symbol types cannot really be differentiated, because types are not handled in a language-agnostic way. return False + + +TSymbolDict = TypeVar("TSymbolDict") +GroupedSymbolDict = dict[str, list[dict] | dict[str, dict]] + + +class SymbolDictGrouper(Generic[TSymbolDict], ABC): + """ + A utility class for grouping a list of symbol dictionaries by one or more specified keys. + + If an instance is statically initialised (upon module import), then this establishes a guarantee + that the specified keys are defined in the symbol dictionary type, ensuring at least basic type safety. + The respective ValueError will immediately be apparent. + """ + + def __init__( + self, + symbol_dict_type: type[TSymbolDict], + children_key: Any, + group_keys: list[Any], + group_children_keys: list[Any], + collapse_singleton: bool, + ) -> None: + """ + :param symbol_dict_type: the TypedDict type that represents the type of the symbol dictionaries to be grouped + :param children_key: the key in the symbol dictionaries that contains the list of child symbols (for recursive grouping). + :param group_keys: keys by which to group the symbol dictionaries. Must be a subset of the keys of `symbol_dict_type`. + :param group_children_keys: keys by which to group the child symbol dictionaries. Must be a subset of the keys of `symbol_dict_type`. + :param collapse_singleton: whether to collapse dictionaries containing a single entry after regrouping to just the entry's value + """ + # check whether the type contains all the keys specified in `keys` and raise an error if not. + if not hasattr(symbol_dict_type, "__annotations__"): + raise ValueError(f"symbol_dict_type must be a TypedDict type, got {symbol_dict_type}") + symbol_dict_keys = set(symbol_dict_type.__annotations__.keys()) + for key in group_keys + [children_key] + group_children_keys: + if key not in symbol_dict_keys: + raise ValueError(f"symbol_dict_type {symbol_dict_type} does not contain key '{key}'") + + self._children_key = children_key + self._group_keys = group_keys + self._group_children_keys = group_children_keys + self._collapse_singleton = collapse_singleton + + def _group_by(self, l: list[dict], keys: list[str], children_keys: list[str]) -> dict[str, Any]: + assert len(keys) > 0, "keys must not be empty" + # group by the first key + grouped: dict[str, Any] = {} + for item in l: + key_value = item.pop(keys[0], "unknown") + if key_value not in grouped: + grouped[key_value] = [] + grouped[key_value].append(item) + if len(keys) > 1: + # continue grouping by the remaining keys + for k, group in grouped.items(): + grouped[k] = self._group_by(group, keys[1:], children_keys) + else: + # grouping is complete; now group the children if necessary + if children_keys: + for k, group in grouped.items(): + for item in group: + if self._children_key in item: + children = item[self._children_key] + item[self._children_key] = self._group_by(children, children_keys, children_keys) + # post-process final group items + grouped = {k: [self._transform_item(i) for i in v] for k, v in grouped.items()} + return grouped + + def _transform_item(self, item: dict) -> dict: + """ + Post-processes a final group item (which has been regrouped, i.e. some keys may have been removed), + collapsing singleton items (and items containing only a single non-children key) + """ + if self._collapse_singleton: + if len(item) == 1: + # {"name": "foo"} -> "foo" + # if there is only a single entry, collapse the dictionary to just the value of that entry + return next(iter(item.values())) + elif len(item) == 2 and self._children_key in item: + # {"name": "foo", "children": {...}} -> {"foo": {...}} + # if there are exactly two entries and one of them is the children key, + # convert to {other_value: children} + other_key = next(k for k in item.keys() if k != self._children_key) + new_item = {item[other_key]: item[self._children_key]} + return new_item + return item + + def group(self, symbols: list[TSymbolDict]) -> GroupedSymbolDict: + """ + :param symbols: the symbols to group + :return: dictionary with the symbols grouped as defined at construction + """ + return self._group_by(symbols, self._group_keys, self._group_children_keys) # type: ignore + + +class LanguageServerSymbolDictGrouper(SymbolDictGrouper[LanguageServerSymbol.OutputDict]): + def __init__( + self, + group_keys: list[LanguageServerSymbol.OutputDictKey], + group_children_keys: list[LanguageServerSymbol.OutputDictKey], + collapse_singleton: bool = False, + ) -> None: + super().__init__(LanguageServerSymbol.OutputDict, "children", group_keys, group_children_keys, collapse_singleton) + + +class JetBrainsSymbolDictGrouper(SymbolDictGrouper[jb.SymbolDTO]): + def __init__( + self, + group_keys: list[jb.SymbolDTOKey], + group_children_keys: list[jb.SymbolDTOKey], + collapse_singleton: bool = False, + map_name_path_to_name: bool = False, + ) -> None: + super().__init__(jb.SymbolDTO, "children", group_keys, group_children_keys, collapse_singleton) + self._map_name_path_to_name = map_name_path_to_name + + def _transform_item(self, item: dict) -> dict: + if self._map_name_path_to_name: + # {"name_path: "Class/myMethod"} -> {"name: "myMethod"} + new_item = dict(item) + if "name_path" in item: + name_path = new_item.pop("name_path") + new_item["name"] = name_path.split("/")[-1] + return super()._transform_item(new_item) + else: + return super()._transform_item(item) diff --git a/src/serena/tools/jetbrains_tools.py b/src/serena/tools/jetbrains_tools.py index 72e0db9c..67f7ac28 100644 --- a/src/serena/tools/jetbrains_tools.py +++ b/src/serena/tools/jetbrains_tools.py @@ -1,10 +1,10 @@ import logging -from collections import defaultdict from typing import Any, Literal -import serena.tools.jetbrains_types as jb +import serena.jetbrains.jetbrains_types as jb +from serena.jetbrains.jetbrains_plugin_client import JetBrainsPluginClient +from serena.symbol import JetBrainsSymbolDictGrouper from serena.tools import Tool, ToolMarkerOptional, ToolMarkerSymbolicRead -from serena.tools.jetbrains_plugin_client import JetBrainsPluginClient log = logging.getLogger(__name__) @@ -91,6 +91,8 @@ class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead, ToolMark Finds symbols that reference the given symbol using the JetBrains backend """ + symbol_dict_grouper = JetBrainsSymbolDictGrouper(["relative_path", "type"], ["type"], collapse_singleton=True) + # TODO: (maybe) - add content snippets showing the references like in LS based version? def apply( self, @@ -118,8 +120,10 @@ class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead, ToolMark relative_path=relative_path, include_quick_info=False, # TODO: Hotfix for serena-jetbrains-plugin/issues/13; revert once fixed ) - result = self._to_json(response_dict) - return self._limit_length(result, max_answer_chars) + symbol_dicts = response_dict["symbols"] + result = self.symbol_dict_grouper.group(symbol_dicts) + result_json = self._to_json(result) + return self._limit_length(result_json, max_answer_chars) class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): @@ -128,37 +132,7 @@ class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOp """ USE_COMPACT_FORMAT = True - - @staticmethod - def _transform_symbols_to_compact_format(symbols: list[jb.SymbolDTO]) -> dict[str, list]: - """ - Transform symbol overview from verbose format to compact grouped format. - - Groups symbols by kind and uses names instead of full symbol objects. - For symbols with children, creates nested dictionaries. - - The name_path can be inferred from the hierarchical structure: - - Top-level symbols: name_path = name - - Nested symbols: name_path = parent_name + "/" + name - For example, "convert" under class "ProjectType" has name_path "ProjectType/convert". - """ - result = defaultdict(list) - - for symbol in symbols: - kind = symbol.get("type", "Unknown") - name_path = symbol["name_path"] - name = name_path.split("/")[-1] - children = symbol.get("children", []) - - if children: - # Symbol has children: create nested dict {name: children_dict} - children_dict = JetBrainsGetSymbolsOverviewTool._transform_symbols_to_compact_format(children) - result[kind].append({name: children_dict}) - else: - # Symbol has no children: just add the name - result[kind].append(name) # type: ignore - - return result + symbol_dict_grouper = JetBrainsSymbolDictGrouper(["type"], ["type"], collapse_singleton=True, map_name_path_to_name=True) def apply( self, @@ -187,7 +161,7 @@ class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOp ) if self.USE_COMPACT_FORMAT: symbols = symbol_overview["symbols"] - result: dict[str, Any] = {"symbols": self._transform_symbols_to_compact_format(symbols)} + result: dict[str, Any] = {"symbols": self.symbol_dict_grouper.group(symbols)} documentation = symbol_overview.pop("documentation", None) if documentation: result["docstring"] = documentation diff --git a/src/serena/tools/symbol_tools.py b/src/serena/tools/symbol_tools.py index ae74bbfd..9a673aeb 100644 --- a/src/serena/tools/symbol_tools.py +++ b/src/serena/tools/symbol_tools.py @@ -3,11 +3,9 @@ Language server-related tools """ import os -from collections import defaultdict from collections.abc import Sequence -from copy import copy -from typing import Any +from serena.symbol import LanguageServerSymbol, LanguageServerSymbolDictGrouper from serena.tools import ( SUCCESS_RESULT, Tool, @@ -18,22 +16,6 @@ from serena.tools.tools_base import ToolMarkerOptional from solidlsp.ls_types import SymbolKind -def _sanitize_symbol_dict(symbol_dict: dict[str, Any]) -> dict[str, Any]: - """ - Sanitize a symbol dictionary inplace by removing unnecessary information. - """ - # We replace the location entry, which repeats line information already included in body_location - # and has unnecessary information on column, by just the relative path. - symbol_dict = copy(symbol_dict) - s_relative_path = symbol_dict.get("location", {}).get("relative_path") - if s_relative_path is not None: - symbol_dict["relative_path"] = s_relative_path - symbol_dict.pop("location", None) - # also remove name, name_path should be enough - symbol_dict.pop("name") - return symbol_dict - - class RestartLanguageServerTool(Tool, ToolMarkerOptional): """Restarts the language server, may be necessary when edits not through Serena happen.""" @@ -50,6 +32,8 @@ class GetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead): Gets an overview of the top-level symbols defined in a given file. """ + symbol_dict_grouper = LanguageServerSymbolDictGrouper(["kind"], ["kind"], collapse_singleton=True) + def apply(self, relative_path: str, depth: int = 0, max_answer_chars: int = -1) -> str: """ Use this tool to get a high-level understanding of the code symbols in a file. @@ -65,11 +49,11 @@ class GetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead): :return: a JSON object containing symbols grouped by kind in a compact format. """ result = self.get_symbol_overview(relative_path, depth=depth) - compact_result = self._transform_symbols_to_compact_format(result) + compact_result = self.symbol_dict_grouper.group(result) result_json_str = self._to_json(compact_result) return self._limit_length(result_json_str, max_answer_chars) - def get_symbol_overview(self, relative_path: str, depth: int = 0) -> list[dict]: + def get_symbol_overview(self, relative_path: str, depth: int = 0) -> list[LanguageServerSymbol.OutputDict]: """ :param relative_path: relative path to a source file :param depth: the depth up to which descendants shall be retrieved @@ -85,37 +69,25 @@ class GetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead): if os.path.isdir(file_path): raise ValueError(f"Expected a file path, but got a directory path: {relative_path}. ") - return symbol_retriever.get_symbol_overview(relative_path, depth=depth)[relative_path] + symbols = symbol_retriever.get_symbol_overview(relative_path)[relative_path] - @staticmethod - def _transform_symbols_to_compact_format(symbols: list[dict[str, Any]]) -> dict[str, list]: - """ - Transform symbol overview from verbose format to compact grouped format. - - Groups symbols by kind and uses names instead of full symbol objects. - For symbols with children, creates nested dictionaries. - - The name_path can be inferred from the hierarchical structure: - - Top-level symbols: name_path = name - - Nested symbols: name_path = parent_name + "/" + name - For example, "convert" under class "ProjectType" has name_path "ProjectType/convert". - """ - result = defaultdict(list) + def child_inclusion_predicate(s: LanguageServerSymbol) -> bool: + return not s.is_low_level() + symbol_dicts = [] for symbol in symbols: - kind = symbol.get("kind", "Unknown") - name = symbol.get("name", "unknown") - children = symbol.get("children", []) - - if children: - # Symbol has children: create nested dict {name: children_dict} - children_dict = GetSymbolsOverviewTool._transform_symbols_to_compact_format(children) - result[kind].append({name: children_dict}) - else: - # Symbol has no children: just add the name - result[kind].append(name) - - return result + symbol_dicts.append( + symbol.to_dict( + name_path=False, + name=True, + depth=depth, + kind=True, + relative_path=False, + location=False, + child_inclusion_predicate=child_inclusion_predicate, + ) + ) + return symbol_dicts class FindSymbolTool(Tool, ToolMarkerSymbolicRead): @@ -185,7 +157,7 @@ class FindSymbolTool(Tool, ToolMarkerSymbolicRead): substring_matching=substring_matching, within_relative_path=relative_path, ) - symbol_dicts = [_sanitize_symbol_dict(s.to_dict(kind=True, location=True, depth=depth, include_body=include_body)) for s in symbols] + symbol_dicts = [dict(s.to_dict(kind=True, relative_path=True, body_location=True, depth=depth, body=include_body)) for s in symbols] if not include_body and include_info: # we add an info field to the symbol dicts if requested for s, s_dict in zip(symbols, symbol_dicts, strict=True): @@ -201,6 +173,8 @@ class FindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead): Finds symbols that reference the given symbol using the language server backend """ + symbol_dict_grouper = LanguageServerSymbolDictGrouper(["relative_path", "kind"], ["kind"], collapse_singleton=True) + # noinspection PyDefaultArgument def apply( self, @@ -229,6 +203,7 @@ class FindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead): parsed_include_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in include_kinds] if include_kinds else None parsed_exclude_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in exclude_kinds] if exclude_kinds else None symbol_retriever = self.create_language_server_symbol_retriever() + references_in_symbols = symbol_retriever.find_referencing_symbols( name_path, relative_file_path=relative_path, @@ -236,10 +211,11 @@ class FindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead): include_kinds=parsed_include_kinds, exclude_kinds=parsed_exclude_kinds, ) + reference_dicts = [] for ref in references_in_symbols: - ref_dict = ref.symbol.to_dict(kind=True, location=True, depth=0, include_body=include_body) - ref_dict = _sanitize_symbol_dict(ref_dict) + ref_dict_orig = ref.symbol.to_dict(kind=True, relative_path=True, depth=0, body=include_body, body_location=True) + ref_dict = dict(ref_dict_orig) if not include_body: ref_relative_path = ref.symbol.location.relative_path assert ref_relative_path is not None, f"Referencing symbol {ref.symbol.name} has no relative path, this is likely a bug." @@ -250,8 +226,11 @@ class FindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead): ) ref_dict["content_around_reference"] = content_around_ref.to_display_string() reference_dicts.append(ref_dict) - result = self._to_json(reference_dicts) - return self._limit_length(result, max_answer_chars) + + result = self.symbol_dict_grouper.group(reference_dicts) # type: ignore + + result_json = self._to_json(result) + return self._limit_length(result_json, max_answer_chars) class ReplaceSymbolBodyTool(Tool, ToolMarkerSymbolicEdit): diff --git a/test/serena/test_serena_agent.py b/test/serena/test_serena_agent.py index 67a8bf14..3a3a8144 100644 --- a/test/serena/test_serena_agent.py +++ b/test/serena/test_serena_agent.py @@ -209,7 +209,7 @@ class TestSerenaAgent: # Find the symbol location first find_symbol_tool = agent.get_tool(FindSymbolTool) - result = find_symbol_tool.apply_ex(name_path_pattern=symbol_name, relative_path=def_file) + result = find_symbol_tool.apply(name_path_pattern=symbol_name, relative_path=def_file) time.sleep(1) symbols = json.loads(result) @@ -218,12 +218,26 @@ class TestSerenaAgent: # Now find references find_refs_tool = agent.get_tool(FindReferencingSymbolsTool) - result = find_refs_tool.apply_ex(name_path=def_symbol["name_path"], relative_path=def_symbol["relative_path"]) + result = find_refs_tool.apply(name_path=def_symbol["name_path"], relative_path=def_symbol["relative_path"]) + + def contains_ref_with_relative_path(refs, relative_path): + """ + Checks for reference to relative path, regardless of output format (grouped an ungrouped) + """ + if isinstance(refs, list): + for ref in refs: + if contains_ref_with_relative_path(ref, relative_path): + return True + elif isinstance(refs, dict): + if relative_path in refs: + return True + for value in refs.values(): + if contains_ref_with_relative_path(value, relative_path): + return True + return False refs = json.loads(result) - assert any( - ref["relative_path"] == ref_file for ref in refs - ), f"Expected to find reference to {symbol_name} in {ref_file}. refs={refs}" + assert contains_ref_with_relative_path(refs, ref_file), f"Expected to find reference to {symbol_name} in {ref_file}. refs={refs}" @pytest.mark.parametrize( "serena_agent,name_path,substring_matching,expected_symbol_name,expected_kind,expected_file", diff --git a/test/serena/test_symbol.py b/test/serena/test_symbol.py index 2e5009de..461d2c94 100644 --- a/test/serena/test_symbol.py +++ b/test/serena/test_symbol.py @@ -1,6 +1,7 @@ import pytest -from serena.symbol import LanguageServerSymbolRetriever, NamePathComponent, NamePathMatcher +from serena.jetbrains.jetbrains_types import SymbolDTO, SymbolDTOKey +from serena.symbol import LanguageServerSymbol, LanguageServerSymbolRetriever, NamePathComponent, NamePathMatcher from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language @@ -238,3 +239,24 @@ class TestLanguageServerSymbolRetriever: create_user_method_symbol = symbol_retriever.find("UserService/create_user", within_relative_path="test_repo/services.py")[0] create_user_method_symbol_info = symbol_retriever.request_info_for_symbol(create_user_method_symbol) assert "Create a new user and store it" in create_user_method_symbol_info + + +class TestSymbolDictTypes: + @staticmethod + def check_key_type(dict_type: type, key_type: type): + """ + :param dict_type: a TypedDict type + :param key_type: the corresponding key type (Literal[...]) that the dict should have for keys + """ + dict_type_keys = dict_type.__annotations__.keys() + assert len(dict_type_keys) == len( + key_type.__args__ # type: ignore + ), f"Expected {len(key_type.__args__)} keys in {dict_type}, but got {len(dict_type_keys)}" # type: ignore + for expected_key in key_type.__args__: # type: ignore + assert expected_key in dict_type_keys, f"Expected key '{expected_key}' not found in {dict_type}" + + def test_ls_symbol_dict_type(self): + self.check_key_type(LanguageServerSymbol.OutputDict, LanguageServerSymbol.OutputDictKey) + + def test_jb_symbol_dict_type(self): + self.check_key_type(SymbolDTO, SymbolDTOKey) diff --git a/test/solidlsp/fsharp/test_fsharp_basic.py b/test/solidlsp/fsharp/test_fsharp_basic.py index 3e599ae5..1d08d82d 100644 --- a/test/solidlsp/fsharp/test_fsharp_basic.py +++ b/test/solidlsp/fsharp/test_fsharp_basic.py @@ -125,6 +125,7 @@ class TestFSharpLanguageServer: # We should get at least some definitions assert len(definitions) >= 0, "Should get definitions (even if empty for complex cases)" + @pytest.mark.skipif(is_ci, reason="Test is flaky") # TODO: Re-enable if the LS can be made more reliable #1039 @pytest.mark.parametrize("language_server", [Language.FSHARP], indirect=True) def test_hover_information(self, language_server: SolidLanguageServer) -> None: """Test hover information functionality.""" diff --git a/test/solidlsp/swift/test_swift_basic.py b/test/solidlsp/swift/test_swift_basic.py index b0f6dc2a..7eae2a16 100644 --- a/test/solidlsp/swift/test_swift_basic.py +++ b/test/solidlsp/swift/test_swift_basic.py @@ -14,6 +14,7 @@ from serena.project import Project from serena.text_utils import LineType from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language +from test.conftest import is_ci # Skip Swift tests on Windows due to complex GitHub Actions configuration WINDOWS_SKIP = platform.system() == "Windows" @@ -99,6 +100,7 @@ class TestSwiftLanguageServerBasics: utils_def = definitions[0] assert utils_def.get("uri", "").endswith("utils.swift"), "Definition should be in utils.swift" + @pytest.mark.skipif(is_ci, reason="Test is flaky in CI") @pytest.mark.parametrize("language_server", [Language.SWIFT], indirect=True) def test_request_references_calculator_class(self, language_server: SolidLanguageServer) -> None: """Test request_references on the Calculator class."""