Remove _sanitize_symbol_dict, making to_dict fully flexible instead

This fixes a bug: _sanitize_symbol_dict was not recursively applied
to children, causing children to contain superfluous entries
(full location, etc.)

Additional change: Improve factorisation of symbol overview computation

src/serena/symbol.py

@@ -396,9 +396,10 @@ class LanguageServerSymbol(Symbol, ToStringMixin):
         return result
 
     class OutputDict(TypedDict):
-        name: str
-        name_path: str
+        name_path: NotRequired[str]
+        name: NotRequired[str]
         location: NotRequired[dict[str, Any]]
+        relative_path: NotRequired[str | None]
         body_location: NotRequired[dict[str, Any]]
         body: NotRequired[str | None]
         kind: NotRequired[str]
@@ -408,17 +409,23 @@ class LanguageServerSymbol(Symbol, ToStringMixin):
 
     def to_dict(
         self,
+        *,
+        name_path: bool = True,
+        name: bool = False,
         kind: bool = False,
         location: bool = False,
         depth: int = 0,
         include_body: bool = False,
+        body_location: bool = False,
         include_children_body: bool = False,
-        include_relative_path: bool = True,
+        include_relative_path: bool = False,
         child_inclusion_predicate: Callable[[Self], bool] | None = None,
     ) -> OutputDict:
         """
         Converts the symbol to a dictionary.
 
+        :param name_path: whether to include the name path of the symbol
+        :param name: whether to include the name of the symbol
         :param kind: whether to include the kind of the symbol
         :param location: whether to include the location of the symbol
         :param depth: the depth up to which to include child symbols (0 = do not include children)
@@ -427,19 +434,30 @@ class LanguageServerSymbol(Symbol, ToStringMixin):
             Note that the body of the children is part of the body of the parent symbol,
             so there is usually no need to set this to True unless you want process the output
             and pass the children without passing the parent body to the LM.
-        :param include_relative_path: whether to include the relative path of the symbol in the location
-            entry. Relative paths of the symbol's children are always excluded.
+        :param include_relative_path: whether to include the relative path of the symbol.
+            If `location` is True, this defines whether to include the path in the location entry.
+            If `location` is False, this defines whether to include the relative path as a top-level entry.
+            Relative paths of the symbol's children are always excluded.
         :param child_inclusion_predicate: an optional predicate that decides whether a child symbol
             should be included.
         :return: a dictionary representation of the symbol
         """
-        result: LanguageServerSymbol.OutputDict = {"name": self.name, "name_path": self.get_name_path()}
+        result: LanguageServerSymbol.OutputDict = {}
+
+        if name_path:
+            result["name_path"] = self.get_name_path()
+        if name:
+            result["name"] = self.name
 
         if kind:
             result["kind"] = self.kind
 
         if location:
             result["location"] = self.location.to_dict(include_relative_path=include_relative_path)
+        elif include_relative_path:
+            result["relative_path"] = self.relative_path
+
+        if body_location:
             body_start_line, body_end_line = self.get_body_line_numbers()
             result["body_location"] = {"start_line": body_start_line, "end_line": body_end_line}
 
@@ -459,8 +477,11 @@ class LanguageServerSymbol(Symbol, ToStringMixin):
                     continue
                 children.append(
                     c.to_dict(
+                        name_path=name_path,
+                        name=name,
                         kind=kind,
                         location=location,
+                        body_location=body_location,
                         depth=depth - 1,
                         child_inclusion_predicate=child_inclusion_predicate,
                         include_body=include_children_body,
@@ -700,36 +721,15 @@ class LanguageServerSymbolRetriever:
 
         return [ReferenceInLanguageServerSymbol.from_lsp_reference(r) for r in references]
 
-    def get_symbol_overview(self, relative_path: str, depth: int = 0) -> dict[str, list[LanguageServerSymbol.OutputDict]]:
+    def get_symbol_overview(self, relative_path: str) -> dict[str, list[LanguageServerSymbol]]:
         """
         :param relative_path: the path of the file or directory for which to get the symbol overview
-        :param depth: the depth up to which to include child symbols (0 = only top-level symbols)
-        :return: a mapping from file paths to lists of symbol dictionaries.
+        :return: a mapping from file paths to lists of symbols.
             For the case where a file is passed, the mapping will contain a single entry.
         """
         lang_server = self.get_language_server(relative_path)
         path_to_unified_symbols = lang_server.request_overview(relative_path)
-
-        def child_inclusion_predicate(s: LanguageServerSymbol) -> bool:
-            return not s.is_low_level()
-
-        result = {}
-        for file_path, unified_symbols in path_to_unified_symbols.items():
-            symbols_in_file = []
-            for us in unified_symbols:
-                symbol = LanguageServerSymbol(us)
-                symbols_in_file.append(
-                    symbol.to_dict(
-                        depth=depth,
-                        kind=True,
-                        include_relative_path=False,
-                        location=False,
-                        child_inclusion_predicate=child_inclusion_predicate,
-                    )
-                )
-            result[file_path] = symbols_in_file
-
-        return result
+        return {k: [LanguageServerSymbol(us) for us in v] for k, v in path_to_unified_symbols.items()}
 
 
 class JetBrainsSymbol(Symbol):

src/serena/tools/symbol_tools.py

@@ -5,8 +5,6 @@ Language server-related tools
 import os
 from collections import defaultdict
 from collections.abc import Sequence
-from copy import copy
-from typing import Any
 
 from serena.symbol import LanguageServerSymbol, LanguageServerSymbolDictGrouper
 from serena.tools import (
@@ -19,23 +17,6 @@ from serena.tools.tools_base import ToolMarkerOptional
 from solidlsp.ls_types import SymbolKind
 
 
-# TODO: We have to get rid of this.
-def _sanitize_symbol_dict(symbol_dict: LanguageServerSymbol.OutputDict) -> dict[str, Any]:
-    """
-    Sanitize a symbol dictionary inplace by removing unnecessary information.
-    """
-    # We replace the location entry, which repeats line information already included in body_location
-    # and has unnecessary information on column, by just the relative path.
-    symbol_dict = copy(symbol_dict)
-    s_relative_path = symbol_dict.get("location", {}).get("relative_path")
-    if s_relative_path is not None:
-        symbol_dict["relative_path"] = s_relative_path
-    symbol_dict.pop("location", None)
-    # also remove name, name_path should be enough
-    symbol_dict.pop("name")
-    return symbol_dict
-
-
 class RestartLanguageServerTool(Tool, ToolMarkerOptional):
     """Restarts the language server, may be necessary when edits not through Serena happen."""
 
@@ -69,8 +50,8 @@ class GetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead):
         :return: a JSON object containing symbols grouped by kind in a compact format.
         """
         result = self.get_symbol_overview(relative_path, depth=depth)
-        # compact_result = self._transform_symbols_to_compact_format(result)
-        compact_result = self.symbol_dict_grouper.group(result)
+        compact_result = self._transform_symbols_to_compact_format(result)
+        # compact_result = self.symbol_dict_grouper.group(result)
         result_json_str = self._to_json(compact_result)
         return self._limit_length(result_json_str, max_answer_chars)
 
@@ -90,10 +71,28 @@ class GetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead):
         if os.path.isdir(file_path):
             raise ValueError(f"Expected a file path, but got a directory path: {relative_path}. ")
 
-        return symbol_retriever.get_symbol_overview(relative_path, depth=depth)[relative_path]
+        symbols = symbol_retriever.get_symbol_overview(relative_path)[relative_path]
+
+        def child_inclusion_predicate(s: LanguageServerSymbol) -> bool:
+            return not s.is_low_level()
+
+        symbol_dicts = []
+        for symbol in symbols:
+            symbol_dicts.append(
+                symbol.to_dict(
+                    name_path=False,
+                    name=True,
+                    depth=depth,
+                    kind=True,
+                    include_relative_path=False,
+                    location=False,
+                    child_inclusion_predicate=child_inclusion_predicate,
+                )
+            )
+        return symbol_dicts
 
     @staticmethod
-    def _transform_symbols_to_compact_format(symbols: list[dict[str, Any]]) -> dict[str, list]:
+    def _transform_symbols_to_compact_format(symbols: list[LanguageServerSymbol.OutputDict]) -> dict[str, list]:
         """
         Transform symbol overview from verbose format to compact grouped format.
 
@@ -190,7 +189,10 @@ class FindSymbolTool(Tool, ToolMarkerSymbolicRead):
             substring_matching=substring_matching,
             within_relative_path=relative_path,
         )
-        symbol_dicts = [_sanitize_symbol_dict(s.to_dict(kind=True, location=True, depth=depth, include_body=include_body)) for s in symbols]
+        symbol_dicts = [
+            dict(s.to_dict(kind=True, include_relative_path=True, body_location=True, depth=depth, include_body=include_body))
+            for s in symbols
+        ]
         if not include_body and include_info:
             # we add an info field to the symbol dicts if requested
             for s, s_dict in zip(symbols, symbol_dicts, strict=True):
@@ -243,8 +245,10 @@ class FindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead):
         )
         reference_dicts = []
         for ref in references_in_symbols:
-            ref_dict = ref.symbol.to_dict(kind=True, location=True, depth=0, include_body=include_body)
-            ref_dict = _sanitize_symbol_dict(ref_dict)
+            ref_dict_orig = ref.symbol.to_dict(
+                kind=True, include_relative_path=True, depth=0, include_body=include_body, body_location=True
+            )
+            ref_dict = dict(ref_dict_orig)
             if not include_body:
                 ref_relative_path = ref.symbol.location.relative_path
                 assert ref_relative_path is not None, f"Referencing symbol {ref.symbol.name} has no relative path, this is likely a bug."