Merge pull request #1036 from oraios/generic-symbol-output-grouping

Implement generic SymbolDictGrouper

src/serena/cli.py

@@ -39,6 +39,7 @@ from serena.tools import FindReferencingSymbolsTool, FindSymbolTool, GetSymbolsO
 from serena.util.dataclass import get_dataclass_default
 from serena.util.logging import MemoryLogHandler
 from solidlsp.ls_config import Language
+from solidlsp.ls_types import SymbolKind
 from solidlsp.util.subprocess_util import subprocess_kwargs
 
 log = logging.getLogger(__name__)
@@ -811,9 +812,7 @@ class ProjectCommands(AutoRegisteringGroup):
                     return
 
                 # Extract suitable symbol (prefer class or function over variables)
-                # LSP symbol kinds: 5=class, 12=function, 6=method, 9=constructor
-                preferred_kinds = [5, 12, 6, 9]  # class, function, method, constructor
-
+                preferred_kinds = {SymbolKind.Class.name, SymbolKind.Function.name, SymbolKind.Method.name, SymbolKind.Constructor.name}
                 selected_symbol = None
                 for symbol in overview_data:
                     if symbol.get("kind") in preferred_kinds:

src/serena/code_editor.py

@@ -6,6 +6,7 @@ from collections.abc import Iterable, Iterator, Reversible
 from contextlib import contextmanager
 from typing import TYPE_CHECKING, Generic, Optional, TypeVar, cast
 
+from serena.jetbrains.jetbrains_plugin_client import JetBrainsPluginClient
 from serena.symbol import JetBrainsSymbol, LanguageServerSymbol, LanguageServerSymbolRetriever, PositionInFile, Symbol
 from solidlsp import SolidLanguageServer, ls_types
 from solidlsp.ls import LSPFileBuffer
@@ -13,7 +14,6 @@ from solidlsp.ls_utils import PathUtils, TextUtils
 
 from .constants import DEFAULT_SOURCE_FILE_ENCODING
 from .project import Project
-from .tools.jetbrains_plugin_client import JetBrainsPluginClient
 
 if TYPE_CHECKING:
     from .agent import SerenaAgent

src/serena/jetbrains/jetbrains_plugin_client.py

@@ -12,10 +12,10 @@ import requests
 from requests import Response
 from sensai.util.string import ToStringMixin
 
-import serena.tools.jetbrains_types as jb
+import serena.jetbrains.jetbrains_types as jb
+from serena.jetbrains.jetbrains_types import PluginStatusDTO
 from serena.project import Project
 from serena.text_utils import render_html
-from serena.tools.jetbrains_types import PluginStatusDTO
 from serena.util.version import Version
 
 T = TypeVar("T")

src/serena/jetbrains/jetbrains_types.py

@@ -1,4 +1,4 @@
-from typing import NotRequired, TypedDict
+from typing import Literal, NotRequired, TypedDict
 
 
 class PluginStatusDTO(TypedDict):
@@ -30,6 +30,9 @@ class SymbolDTO(TypedDict):
     num_usages: NotRequired[int]
 
 
+SymbolDTOKey = Literal["name_path", "relative_path", "type", "body", "quick_info", "documentation", "text_range", "children", "num_usages"]
+
+
 class SymbolCollectionResponse(TypedDict):
     symbols: list[SymbolDTO]
 

src/serena/symbol.py

@@ -4,11 +4,11 @@ import os
 from abc import ABC, abstractmethod
 from collections.abc import Callable, Iterator, Sequence
 from dataclasses import asdict, dataclass
-from typing import TYPE_CHECKING, Any, Self, Union
+from typing import TYPE_CHECKING, Any, Generic, Literal, NotRequired, Self, TypedDict, TypeVar, Union
 
 from sensai.util.string import ToStringMixin
 
-import serena.tools.jetbrains_types as jb
+import serena.jetbrains.jetbrains_types as jb
 from solidlsp import SolidLanguageServer
 from solidlsp.ls import ReferenceInSymbol as LSPReferenceInSymbol
 from solidlsp.ls_types import Position, SymbolKind, UnifiedSymbolInformation
@@ -212,14 +212,17 @@ class LanguageServerSymbol(Symbol, ToStringMixin):
         return []
 
     def _tostring_additional_entries(self) -> dict[str, Any]:
-        return dict(name=self.name, kind=self.kind, num_children=len(self.symbol_root["children"]))
+        return dict(name=self.name, kind=self.symbol_kind_name, num_children=len(self.symbol_root["children"]))
 
     @property
     def name(self) -> str:
         return self.symbol_root["name"]
 
     @property
-    def kind(self) -> str:
+    def symbol_kind_name(self) -> str:
+        """
+        :return: string representation of the symbol kind (name attribute of the `SymbolKind` enum item)
+        """
         return SymbolKind(self.symbol_kind).name
 
     @property
@@ -395,67 +398,99 @@ class LanguageServerSymbol(Symbol, ToStringMixin):
         traverse(self)
         return result
 
+    class OutputDict(TypedDict):
+        name_path: NotRequired[str]
+        name: NotRequired[str]
+        location: NotRequired[dict[str, Any]]
+        relative_path: NotRequired[str | None]
+        body_location: NotRequired[dict[str, Any]]
+        body: NotRequired[str | None]
+        kind: NotRequired[str]
+        """
+        string representation of the symbol kind (name attribute of the `SymbolKind` enum item)
+        """
+        children: NotRequired[list["LanguageServerSymbol.OutputDict"]]
+
+    OutputDictKey = Literal["name", "name_path", "relative_path", "location", "body_location", "body", "kind", "children"]
+
     def to_dict(
         self,
+        *,
+        name_path: bool = True,
+        name: bool = False,
         kind: bool = False,
         location: bool = False,
         depth: int = 0,
-        include_body: bool = False,
-        include_children_body: bool = False,
-        include_relative_path: bool = True,
+        body: bool = False,
+        body_location: bool = False,
+        children_body: bool = False,
+        relative_path: bool = False,
         child_inclusion_predicate: Callable[[Self], bool] | None = None,
-    ) -> dict[str, Any]:
+    ) -> OutputDict:
         """
         Converts the symbol to a dictionary.
 
+        :param name_path: whether to include the name path of the symbol
+        :param name: whether to include the name of the symbol
         :param kind: whether to include the kind of the symbol
         :param location: whether to include the location of the symbol
         :param depth: the depth up to which to include child symbols (0 = do not include children)
-        :param include_body: whether to include the body of the top-level symbol.
-        :param include_children_body: whether to also include the body of the children.
+        :param body: whether to include the body of the top-level symbol.
+        :param children_body: whether to also include the body of the children.
             Note that the body of the children is part of the body of the parent symbol,
             so there is usually no need to set this to True unless you want process the output
             and pass the children without passing the parent body to the LM.
-        :param include_relative_path: whether to include the relative path of the symbol in the location
-            entry. Relative paths of the symbol's children are always excluded.
+        :param relative_path: whether to include the relative path of the symbol.
+            If `location` is True, this defines whether to include the path in the location entry.
+            If `location` is False, this defines whether to include the relative path as a top-level entry.
+            Relative paths of the symbol's children are always excluded.
         :param child_inclusion_predicate: an optional predicate that decides whether a child symbol
             should be included.
         :return: a dictionary representation of the symbol
         """
-        result: dict[str, Any] = {"name": self.name, "name_path": self.get_name_path()}
+        result: LanguageServerSymbol.OutputDict = {}
+
+        if name_path:
+            result["name_path"] = self.get_name_path()
+        if name:
+            result["name"] = self.name
 
         if kind:
-            result["kind"] = self.kind
+            result["kind"] = self.symbol_kind_name
 
         if location:
-            result["location"] = self.location.to_dict(include_relative_path=include_relative_path)
+            result["location"] = self.location.to_dict(include_relative_path=relative_path)
+        elif relative_path:
+            result["relative_path"] = self.relative_path
+
+        if body_location:
             body_start_line, body_end_line = self.get_body_line_numbers()
             result["body_location"] = {"start_line": body_start_line, "end_line": body_end_line}
 
-        if include_body:
-            body = self.body
-            if body is None:
-                log.warning("Requested body for symbol, but it is not present. The symbol might have been loaded with include_body=False.")
-            result["body"] = body
+        if body:
+            result["body"] = self.body
 
         if child_inclusion_predicate is None:
             child_inclusion_predicate = lambda s: True
 
-        def included_children(s: Self) -> list[dict[str, Any]]:
+        def included_children(s: Self) -> list[LanguageServerSymbol.OutputDict]:
             children = []
             for c in s.iter_children():
                 if not child_inclusion_predicate(c):
                     continue
                 children.append(
                     c.to_dict(
+                        name_path=name_path,
+                        name=name,
                         kind=kind,
                         location=location,
+                        body_location=body_location,
                         depth=depth - 1,
                         child_inclusion_predicate=child_inclusion_predicate,
-                        include_body=include_children_body,
-                        include_children_body=include_children_body,
+                        body=children_body,
+                        children_body=children_body,
                         # all children have the same relative path as the parent
-                        include_relative_path=False,
+                        relative_path=False,
                     )
                 )
             return children
@@ -463,7 +498,7 @@ class LanguageServerSymbol(Symbol, ToStringMixin):
         if depth > 0:
             children = included_children(self)
             if len(children) > 0:
-                result["children"] = included_children(self)
+                result["children"] = children
 
         return result
 
@@ -604,8 +639,7 @@ class LanguageServerSymbolRetriever:
             include_rel_path = within_relative_path is not None
             raise ValueError(
                 f"Found multiple {len(symbol_candidates)} symbols matching '{name_path_pattern}'. "
-                "They are: \n"
-                + json.dumps([s.to_dict(kind=True, include_relative_path=include_rel_path) for s in symbol_candidates], indent=2)
+                "They are: \n" + json.dumps([s.to_dict(kind=True, relative_path=include_rel_path) for s in symbol_candidates], indent=2)
             )
 
     def find_by_location(self, location: LanguageServerSymbolLocation) -> LanguageServerSymbol | None:
@@ -689,36 +723,15 @@ class LanguageServerSymbolRetriever:
 
         return [ReferenceInLanguageServerSymbol.from_lsp_reference(r) for r in references]
 
-    def get_symbol_overview(self, relative_path: str, depth: int = 0) -> dict[str, list[dict]]:
+    def get_symbol_overview(self, relative_path: str) -> dict[str, list[LanguageServerSymbol]]:
         """
         :param relative_path: the path of the file or directory for which to get the symbol overview
-        :param depth: the depth up to which to include child symbols (0 = only top-level symbols)
-        :return: a mapping from file paths to lists of symbol dictionaries.
+        :return: a mapping from file paths to lists of symbols.
             For the case where a file is passed, the mapping will contain a single entry.
         """
         lang_server = self.get_language_server(relative_path)
         path_to_unified_symbols = lang_server.request_overview(relative_path)
-
-        def child_inclusion_predicate(s: LanguageServerSymbol) -> bool:
-            return not s.is_low_level()
-
-        result = {}
-        for file_path, unified_symbols in path_to_unified_symbols.items():
-            symbols_in_file = []
-            for us in unified_symbols:
-                symbol = LanguageServerSymbol(us)
-                symbols_in_file.append(
-                    symbol.to_dict(
-                        depth=depth,
-                        kind=True,
-                        include_relative_path=False,
-                        location=False,
-                        child_inclusion_predicate=child_inclusion_predicate,
-                    )
-                )
-            result[file_path] = symbols_in_file
-
-        return result
+        return {k: [LanguageServerSymbol(us) for us in v] for k, v in path_to_unified_symbols.items()}
 
 
 class JetBrainsSymbol(Symbol):
@@ -775,3 +788,129 @@ class JetBrainsSymbol(Symbol):
     def is_neighbouring_definition_separated_by_empty_line(self) -> bool:
         # NOTE: Symbol types cannot really be differentiated, because types are not handled in a language-agnostic way.
         return False
+
+
+TSymbolDict = TypeVar("TSymbolDict")
+GroupedSymbolDict = dict[str, list[dict] | dict[str, dict]]
+
+
+class SymbolDictGrouper(Generic[TSymbolDict], ABC):
+    """
+    A utility class for grouping a list of symbol dictionaries by one or more specified keys.
+
+    If an instance is statically initialised (upon module import), then this establishes a guarantee
+    that the specified keys are defined in the symbol dictionary type, ensuring at least basic type safety.
+    The respective ValueError will immediately be apparent.
+    """
+
+    def __init__(
+        self,
+        symbol_dict_type: type[TSymbolDict],
+        children_key: Any,
+        group_keys: list[Any],
+        group_children_keys: list[Any],
+        collapse_singleton: bool,
+    ) -> None:
+        """
+        :param symbol_dict_type: the TypedDict type that represents the type of the symbol dictionaries to be grouped
+        :param children_key: the key in the symbol dictionaries that contains the list of child symbols (for recursive grouping).
+        :param group_keys: keys by which to group the symbol dictionaries. Must be a subset of the keys of `symbol_dict_type`.
+        :param group_children_keys: keys by which to group the child symbol dictionaries. Must be a subset of the keys of `symbol_dict_type`.
+        :param collapse_singleton: whether to collapse dictionaries containing a single entry after regrouping to just the entry's value
+        """
+        # check whether the type contains all the keys specified in `keys` and raise an error if not.
+        if not hasattr(symbol_dict_type, "__annotations__"):
+            raise ValueError(f"symbol_dict_type must be a TypedDict type, got {symbol_dict_type}")
+        symbol_dict_keys = set(symbol_dict_type.__annotations__.keys())
+        for key in group_keys + [children_key] + group_children_keys:
+            if key not in symbol_dict_keys:
+                raise ValueError(f"symbol_dict_type {symbol_dict_type} does not contain key '{key}'")
+
+        self._children_key = children_key
+        self._group_keys = group_keys
+        self._group_children_keys = group_children_keys
+        self._collapse_singleton = collapse_singleton
+
+    def _group_by(self, l: list[dict], keys: list[str], children_keys: list[str]) -> dict[str, Any]:
+        assert len(keys) > 0, "keys must not be empty"
+        # group by the first key
+        grouped: dict[str, Any] = {}
+        for item in l:
+            key_value = item.pop(keys[0], "unknown")
+            if key_value not in grouped:
+                grouped[key_value] = []
+            grouped[key_value].append(item)
+        if len(keys) > 1:
+            # continue grouping by the remaining keys
+            for k, group in grouped.items():
+                grouped[k] = self._group_by(group, keys[1:], children_keys)
+        else:
+            # grouping is complete; now group the children if necessary
+            if children_keys:
+                for k, group in grouped.items():
+                    for item in group:
+                        if self._children_key in item:
+                            children = item[self._children_key]
+                            item[self._children_key] = self._group_by(children, children_keys, children_keys)
+            # post-process final group items
+            grouped = {k: [self._transform_item(i) for i in v] for k, v in grouped.items()}
+        return grouped
+
+    def _transform_item(self, item: dict) -> dict:
+        """
+        Post-processes a final group item (which has been regrouped, i.e. some keys may have been removed),
+        collapsing singleton items (and items containing only a single non-children key)
+        """
+        if self._collapse_singleton:
+            if len(item) == 1:
+                # {"name": "foo"} -> "foo"
+                # if there is only a single entry, collapse the dictionary to just the value of that entry
+                return next(iter(item.values()))
+            elif len(item) == 2 and self._children_key in item:
+                # {"name": "foo", "children": {...}} -> {"foo": {...}}
+                # if there are exactly two entries and one of them is the children key,
+                # convert to {other_value: children}
+                other_key = next(k for k in item.keys() if k != self._children_key)
+                new_item = {item[other_key]: item[self._children_key]}
+                return new_item
+        return item
+
+    def group(self, symbols: list[TSymbolDict]) -> GroupedSymbolDict:
+        """
+        :param symbols: the symbols to group
+        :return: dictionary with the symbols grouped as defined at construction
+        """
+        return self._group_by(symbols, self._group_keys, self._group_children_keys)  # type: ignore
+
+
+class LanguageServerSymbolDictGrouper(SymbolDictGrouper[LanguageServerSymbol.OutputDict]):
+    def __init__(
+        self,
+        group_keys: list[LanguageServerSymbol.OutputDictKey],
+        group_children_keys: list[LanguageServerSymbol.OutputDictKey],
+        collapse_singleton: bool = False,
+    ) -> None:
+        super().__init__(LanguageServerSymbol.OutputDict, "children", group_keys, group_children_keys, collapse_singleton)
+
+
+class JetBrainsSymbolDictGrouper(SymbolDictGrouper[jb.SymbolDTO]):
+    def __init__(
+        self,
+        group_keys: list[jb.SymbolDTOKey],
+        group_children_keys: list[jb.SymbolDTOKey],
+        collapse_singleton: bool = False,
+        map_name_path_to_name: bool = False,
+    ) -> None:
+        super().__init__(jb.SymbolDTO, "children", group_keys, group_children_keys, collapse_singleton)
+        self._map_name_path_to_name = map_name_path_to_name
+
+    def _transform_item(self, item: dict) -> dict:
+        if self._map_name_path_to_name:
+            # {"name_path: "Class/myMethod"} -> {"name: "myMethod"}
+            new_item = dict(item)
+            if "name_path" in item:
+                name_path = new_item.pop("name_path")
+                new_item["name"] = name_path.split("/")[-1]
+            return super()._transform_item(new_item)
+        else:
+            return super()._transform_item(item)

src/serena/tools/jetbrains_tools.py

@@ -1,10 +1,10 @@
 import logging
-from collections import defaultdict
 from typing import Any, Literal
 
-import serena.tools.jetbrains_types as jb
+import serena.jetbrains.jetbrains_types as jb
+from serena.jetbrains.jetbrains_plugin_client import JetBrainsPluginClient
+from serena.symbol import JetBrainsSymbolDictGrouper
 from serena.tools import Tool, ToolMarkerOptional, ToolMarkerSymbolicRead
-from serena.tools.jetbrains_plugin_client import JetBrainsPluginClient
 
 log = logging.getLogger(__name__)
 
@@ -91,6 +91,8 @@ class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead, ToolMark
     Finds symbols that reference the given symbol using the JetBrains backend
     """
 
+    symbol_dict_grouper = JetBrainsSymbolDictGrouper(["relative_path", "type"], ["type"], collapse_singleton=True)
+
     # TODO: (maybe) - add content snippets showing the references like in LS based version?
     def apply(
         self,
@@ -118,8 +120,10 @@ class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead, ToolMark
                 relative_path=relative_path,
                 include_quick_info=False,  # TODO: Hotfix for serena-jetbrains-plugin/issues/13; revert once fixed
             )
-            result = self._to_json(response_dict)
-        return self._limit_length(result, max_answer_chars)
+        symbol_dicts = response_dict["symbols"]
+        result = self.symbol_dict_grouper.group(symbol_dicts)
+        result_json = self._to_json(result)
+        return self._limit_length(result_json, max_answer_chars)
 
 
 class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional):
@@ -128,37 +132,7 @@ class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOp
     """
 
     USE_COMPACT_FORMAT = True
-
-    @staticmethod
-    def _transform_symbols_to_compact_format(symbols: list[jb.SymbolDTO]) -> dict[str, list]:
-        """
-        Transform symbol overview from verbose format to compact grouped format.
-
-        Groups symbols by kind and uses names instead of full symbol objects.
-        For symbols with children, creates nested dictionaries.
-
-        The name_path can be inferred from the hierarchical structure:
-        - Top-level symbols: name_path = name
-        - Nested symbols: name_path = parent_name + "/" + name
-        For example, "convert" under class "ProjectType" has name_path "ProjectType/convert".
-        """
-        result = defaultdict(list)
-
-        for symbol in symbols:
-            kind = symbol.get("type", "Unknown")
-            name_path = symbol["name_path"]
-            name = name_path.split("/")[-1]
-            children = symbol.get("children", [])
-
-            if children:
-                # Symbol has children: create nested dict {name: children_dict}
-                children_dict = JetBrainsGetSymbolsOverviewTool._transform_symbols_to_compact_format(children)
-                result[kind].append({name: children_dict})
-            else:
-                # Symbol has no children: just add the name
-                result[kind].append(name)  # type: ignore
-
-        return result
+    symbol_dict_grouper = JetBrainsSymbolDictGrouper(["type"], ["type"], collapse_singleton=True, map_name_path_to_name=True)
 
     def apply(
         self,
@@ -187,7 +161,7 @@ class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOp
             )
         if self.USE_COMPACT_FORMAT:
             symbols = symbol_overview["symbols"]
-            result: dict[str, Any] = {"symbols": self._transform_symbols_to_compact_format(symbols)}
+            result: dict[str, Any] = {"symbols": self.symbol_dict_grouper.group(symbols)}
             documentation = symbol_overview.pop("documentation", None)
             if documentation:
                 result["docstring"] = documentation

src/serena/tools/symbol_tools.py

@@ -3,11 +3,9 @@ Language server-related tools
 """
 
 import os
-from collections import defaultdict
 from collections.abc import Sequence
-from copy import copy
-from typing import Any
 
+from serena.symbol import LanguageServerSymbol, LanguageServerSymbolDictGrouper
 from serena.tools import (
     SUCCESS_RESULT,
     Tool,
@@ -18,22 +16,6 @@ from serena.tools.tools_base import ToolMarkerOptional
 from solidlsp.ls_types import SymbolKind
 
 
-def _sanitize_symbol_dict(symbol_dict: dict[str, Any]) -> dict[str, Any]:
-    """
-    Sanitize a symbol dictionary inplace by removing unnecessary information.
-    """
-    # We replace the location entry, which repeats line information already included in body_location
-    # and has unnecessary information on column, by just the relative path.
-    symbol_dict = copy(symbol_dict)
-    s_relative_path = symbol_dict.get("location", {}).get("relative_path")
-    if s_relative_path is not None:
-        symbol_dict["relative_path"] = s_relative_path
-    symbol_dict.pop("location", None)
-    # also remove name, name_path should be enough
-    symbol_dict.pop("name")
-    return symbol_dict
-
-
 class RestartLanguageServerTool(Tool, ToolMarkerOptional):
     """Restarts the language server, may be necessary when edits not through Serena happen."""
 
@@ -50,6 +32,8 @@ class GetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead):
     Gets an overview of the top-level symbols defined in a given file.
     """
 
+    symbol_dict_grouper = LanguageServerSymbolDictGrouper(["kind"], ["kind"], collapse_singleton=True)
+
     def apply(self, relative_path: str, depth: int = 0, max_answer_chars: int = -1) -> str:
         """
         Use this tool to get a high-level understanding of the code symbols in a file.
@@ -65,11 +49,11 @@ class GetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead):
         :return: a JSON object containing symbols grouped by kind in a compact format.
         """
         result = self.get_symbol_overview(relative_path, depth=depth)
-        compact_result = self._transform_symbols_to_compact_format(result)
+        compact_result = self.symbol_dict_grouper.group(result)
         result_json_str = self._to_json(compact_result)
         return self._limit_length(result_json_str, max_answer_chars)
 
-    def get_symbol_overview(self, relative_path: str, depth: int = 0) -> list[dict]:
+    def get_symbol_overview(self, relative_path: str, depth: int = 0) -> list[LanguageServerSymbol.OutputDict]:
         """
         :param relative_path: relative path to a source file
         :param depth: the depth up to which descendants shall be retrieved
@@ -85,37 +69,25 @@ class GetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead):
         if os.path.isdir(file_path):
             raise ValueError(f"Expected a file path, but got a directory path: {relative_path}. ")
 
-        return symbol_retriever.get_symbol_overview(relative_path, depth=depth)[relative_path]
+        symbols = symbol_retriever.get_symbol_overview(relative_path)[relative_path]
 
-    @staticmethod
-    def _transform_symbols_to_compact_format(symbols: list[dict[str, Any]]) -> dict[str, list]:
-        """
-        Transform symbol overview from verbose format to compact grouped format.
-
-        Groups symbols by kind and uses names instead of full symbol objects.
-        For symbols with children, creates nested dictionaries.
-
-        The name_path can be inferred from the hierarchical structure:
-        - Top-level symbols: name_path = name
-        - Nested symbols: name_path = parent_name + "/" + name
-        For example, "convert" under class "ProjectType" has name_path "ProjectType/convert".
-        """
-        result = defaultdict(list)
+        def child_inclusion_predicate(s: LanguageServerSymbol) -> bool:
+            return not s.is_low_level()
 
+        symbol_dicts = []
         for symbol in symbols:
-            kind = symbol.get("kind", "Unknown")
-            name = symbol.get("name", "unknown")
-            children = symbol.get("children", [])
-
-            if children:
-                # Symbol has children: create nested dict {name: children_dict}
-                children_dict = GetSymbolsOverviewTool._transform_symbols_to_compact_format(children)
-                result[kind].append({name: children_dict})
-            else:
-                # Symbol has no children: just add the name
-                result[kind].append(name)
-
-        return result
+            symbol_dicts.append(
+                symbol.to_dict(
+                    name_path=False,
+                    name=True,
+                    depth=depth,
+                    kind=True,
+                    relative_path=False,
+                    location=False,
+                    child_inclusion_predicate=child_inclusion_predicate,
+                )
+            )
+        return symbol_dicts
 
 
 class FindSymbolTool(Tool, ToolMarkerSymbolicRead):
@@ -185,7 +157,7 @@ class FindSymbolTool(Tool, ToolMarkerSymbolicRead):
             substring_matching=substring_matching,
             within_relative_path=relative_path,
         )
-        symbol_dicts = [_sanitize_symbol_dict(s.to_dict(kind=True, location=True, depth=depth, include_body=include_body)) for s in symbols]
+        symbol_dicts = [dict(s.to_dict(kind=True, relative_path=True, body_location=True, depth=depth, body=include_body)) for s in symbols]
         if not include_body and include_info:
             # we add an info field to the symbol dicts if requested
             for s, s_dict in zip(symbols, symbol_dicts, strict=True):
@@ -201,6 +173,8 @@ class FindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead):
     Finds symbols that reference the given symbol using the language server backend
     """
 
+    symbol_dict_grouper = LanguageServerSymbolDictGrouper(["relative_path", "kind"], ["kind"], collapse_singleton=True)
+
     # noinspection PyDefaultArgument
     def apply(
         self,
@@ -229,6 +203,7 @@ class FindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead):
         parsed_include_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in include_kinds] if include_kinds else None
         parsed_exclude_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in exclude_kinds] if exclude_kinds else None
         symbol_retriever = self.create_language_server_symbol_retriever()
+
         references_in_symbols = symbol_retriever.find_referencing_symbols(
             name_path,
             relative_file_path=relative_path,
@@ -236,10 +211,11 @@ class FindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead):
             include_kinds=parsed_include_kinds,
             exclude_kinds=parsed_exclude_kinds,
         )
+
         reference_dicts = []
         for ref in references_in_symbols:
-            ref_dict = ref.symbol.to_dict(kind=True, location=True, depth=0, include_body=include_body)
-            ref_dict = _sanitize_symbol_dict(ref_dict)
+            ref_dict_orig = ref.symbol.to_dict(kind=True, relative_path=True, depth=0, body=include_body, body_location=True)
+            ref_dict = dict(ref_dict_orig)
             if not include_body:
                 ref_relative_path = ref.symbol.location.relative_path
                 assert ref_relative_path is not None, f"Referencing symbol {ref.symbol.name} has no relative path, this is likely a bug."
@@ -250,8 +226,11 @@ class FindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead):
                 )
                 ref_dict["content_around_reference"] = content_around_ref.to_display_string()
             reference_dicts.append(ref_dict)
-        result = self._to_json(reference_dicts)
-        return self._limit_length(result, max_answer_chars)
+
+        result = self.symbol_dict_grouper.group(reference_dicts)  # type: ignore
+
+        result_json = self._to_json(result)
+        return self._limit_length(result_json, max_answer_chars)
 
 
 class ReplaceSymbolBodyTool(Tool, ToolMarkerSymbolicEdit):

test/serena/test_serena_agent.py

@@ -209,7 +209,7 @@ class TestSerenaAgent:
 
         # Find the symbol location first
         find_symbol_tool = agent.get_tool(FindSymbolTool)
-        result = find_symbol_tool.apply_ex(name_path_pattern=symbol_name, relative_path=def_file)
+        result = find_symbol_tool.apply(name_path_pattern=symbol_name, relative_path=def_file)
 
         time.sleep(1)
         symbols = json.loads(result)
@@ -218,12 +218,26 @@ class TestSerenaAgent:
 
         # Now find references
         find_refs_tool = agent.get_tool(FindReferencingSymbolsTool)
-        result = find_refs_tool.apply_ex(name_path=def_symbol["name_path"], relative_path=def_symbol["relative_path"])
+        result = find_refs_tool.apply(name_path=def_symbol["name_path"], relative_path=def_symbol["relative_path"])
+
+        def contains_ref_with_relative_path(refs, relative_path):
+            """
+            Checks for reference to relative path, regardless of output format (grouped an ungrouped)
+            """
+            if isinstance(refs, list):
+                for ref in refs:
+                    if contains_ref_with_relative_path(ref, relative_path):
+                        return True
+            elif isinstance(refs, dict):
+                if relative_path in refs:
+                    return True
+                for value in refs.values():
+                    if contains_ref_with_relative_path(value, relative_path):
+                        return True
+            return False
 
         refs = json.loads(result)
-        assert any(
-            ref["relative_path"] == ref_file for ref in refs
-        ), f"Expected to find reference to {symbol_name} in {ref_file}. refs={refs}"
+        assert contains_ref_with_relative_path(refs, ref_file), f"Expected to find reference to {symbol_name} in {ref_file}. refs={refs}"
 
     @pytest.mark.parametrize(
         "serena_agent,name_path,substring_matching,expected_symbol_name,expected_kind,expected_file",

test/serena/test_symbol.py

@@ -1,6 +1,7 @@
 import pytest
 
-from serena.symbol import LanguageServerSymbolRetriever, NamePathComponent, NamePathMatcher
+from serena.jetbrains.jetbrains_types import SymbolDTO, SymbolDTOKey
+from serena.symbol import LanguageServerSymbol, LanguageServerSymbolRetriever, NamePathComponent, NamePathMatcher
 from solidlsp import SolidLanguageServer
 from solidlsp.ls_config import Language
 
@@ -238,3 +239,24 @@ class TestLanguageServerSymbolRetriever:
         create_user_method_symbol = symbol_retriever.find("UserService/create_user", within_relative_path="test_repo/services.py")[0]
         create_user_method_symbol_info = symbol_retriever.request_info_for_symbol(create_user_method_symbol)
         assert "Create a new user and store it" in create_user_method_symbol_info
+
+
+class TestSymbolDictTypes:
+    @staticmethod
+    def check_key_type(dict_type: type, key_type: type):
+        """
+        :param dict_type: a TypedDict type
+        :param key_type: the corresponding key type (Literal[...]) that the dict should have for keys
+        """
+        dict_type_keys = dict_type.__annotations__.keys()
+        assert len(dict_type_keys) == len(
+            key_type.__args__  # type: ignore
+        ), f"Expected {len(key_type.__args__)} keys in {dict_type}, but got {len(dict_type_keys)}"  # type: ignore
+        for expected_key in key_type.__args__:  # type: ignore
+            assert expected_key in dict_type_keys, f"Expected key '{expected_key}' not found in {dict_type}"
+
+    def test_ls_symbol_dict_type(self):
+        self.check_key_type(LanguageServerSymbol.OutputDict, LanguageServerSymbol.OutputDictKey)
+
+    def test_jb_symbol_dict_type(self):
+        self.check_key_type(SymbolDTO, SymbolDTOKey)

test/solidlsp/fsharp/test_fsharp_basic.py

@@ -125,6 +125,7 @@ class TestFSharpLanguageServer:
         # We should get at least some definitions
         assert len(definitions) >= 0, "Should get definitions (even if empty for complex cases)"
 
+    @pytest.mark.skipif(is_ci, reason="Test is flaky")  # TODO: Re-enable if the LS can be made more reliable #1039
     @pytest.mark.parametrize("language_server", [Language.FSHARP], indirect=True)
     def test_hover_information(self, language_server: SolidLanguageServer) -> None:
         """Test hover information functionality."""

test/solidlsp/swift/test_swift_basic.py

@@ -14,6 +14,7 @@ from serena.project import Project
 from serena.text_utils import LineType
 from solidlsp import SolidLanguageServer
 from solidlsp.ls_config import Language
+from test.conftest import is_ci
 
 # Skip Swift tests on Windows due to complex GitHub Actions configuration
 WINDOWS_SKIP = platform.system() == "Windows"
@@ -99,6 +100,7 @@ class TestSwiftLanguageServerBasics:
         utils_def = definitions[0]
         assert utils_def.get("uri", "").endswith("utils.swift"), "Definition should be in utils.swift"
 
+    @pytest.mark.skipif(is_ci, reason="Test is flaky in CI")
     @pytest.mark.parametrize("language_server", [Language.SWIFT], indirect=True)
     def test_request_references_calculator_class(self, language_server: SolidLanguageServer) -> None:
         """Test request_references on the Calculator class."""