Support for info in JB symbols and various improvements. Adds bs4 to deps

- Added typed dicts for most responses in jetbrains_types.py
- Fix bug in code_editor (location key no longer existed, found by mypy after adding the types)

Should be merged after next version of plugin release

pyproject.toml

@@ -35,6 +35,7 @@ dependencies = [
   "tqdm>=4.67.1",
   "tiktoken>=0.9.0",
   "anthropic>=0.54.0",
+  "beautifulsoup4>=4.14.2",
 ]
 
 [[tool.uv.index]]
@@ -314,4 +315,4 @@ markers = [
 skip = '.git*,*.svg,*.lock,*.min.*'
 check-hidden = true
 # ignore-regex = ''
-ignore-words-list = 'paket'
+ignore-words-list = 'paket'

scripts/demo_run_tools.py

@@ -7,21 +7,30 @@ import json
 from pprint import pprint
 
 from serena.agent import SerenaAgent
+from serena.config.serena_config import SerenaConfig
 from serena.constants import REPO_ROOT
-from serena.tools import FindFileTool, FindReferencingSymbolsTool, GetSymbolsOverviewTool, JetBrainsFindSymbolTool, SearchForPatternTool
+from serena.tools import (
+    FindFileTool,
+    FindReferencingSymbolsTool,
+    JetBrainsFindSymbolTool,
+    JetBrainsGetSymbolsOverviewTool,
+    SearchForPatternTool,
+)
 
 if __name__ == "__main__":
-    agent = SerenaAgent(project=REPO_ROOT)
+    serena_config = SerenaConfig.from_config_file()
+    serena_config.web_dashboard = False
+    agent = SerenaAgent(project=REPO_ROOT, serena_config=serena_config)
 
     # apply a tool
     find_symbol_tool = agent.get_tool(JetBrainsFindSymbolTool)
     find_refs_tool = agent.get_tool(FindReferencingSymbolsTool)
     find_file_tool = agent.get_tool(FindFileTool)
     search_pattern_tool = agent.get_tool(SearchForPatternTool)
-    overview_tool = agent.get_tool(GetSymbolsOverviewTool)
+    overview_tool = agent.get_tool(JetBrainsGetSymbolsOverviewTool)
 
     result = agent.execute_task(
-        lambda: find_symbol_tool.apply("displayBasicStats"),
+        lambda: find_symbol_tool.apply("SerenaAgent/get_tool_description_override"),
     )
     pprint(json.loads(result))
     # input("Press Enter to continue...")

src/serena/code_editor.py

@@ -422,7 +422,7 @@ class JetBrainsCodeEditor(CodeEditor[JetBrainsSymbol]):
             if len(symbols) > 1:
                 raise ValueError(
                     f"Found multiple {len(symbols)} symbols with name {name_path} in file {relative_file_path}. "
-                    "Their locations are: \n " + json.dumps([s["location"] for s in symbols], indent=2)
+                    "Their locations are: \n " + json.dumps([s["text_range"] for s in symbols], indent=2)
                 )
             return JetBrainsSymbol(symbols[0], self._project)
 

src/serena/symbol.py

@@ -4,10 +4,11 @@ import os
 from abc import ABC, abstractmethod
 from collections.abc import Callable, Iterator, Sequence
 from dataclasses import asdict, dataclass
-from typing import TYPE_CHECKING, Any, NotRequired, Self, TypedDict, Union
+from typing import TYPE_CHECKING, Any, Self, Union
 
 from sensai.util.string import ToStringMixin
 
+import serena.tools.jetbrain_types as jb
 from solidlsp import SolidLanguageServer
 from solidlsp.ls import ReferenceInSymbol as LSPReferenceInSymbol
 from solidlsp.ls_types import Position, SymbolKind, UnifiedSymbolInformation
@@ -705,15 +706,7 @@ class LanguageServerSymbolRetriever:
 
 
 class JetBrainsSymbol(Symbol):
-    class SymbolDict(TypedDict):
-        name_path: str
-        relative_path: str
-        type: str
-        text_range: NotRequired[dict]
-        body: NotRequired[str]
-        children: NotRequired[list["JetBrainsSymbol.SymbolDict"]]
-
-    def __init__(self, symbol_dict: SymbolDict, project: Project) -> None:
+    def __init__(self, symbol_dict: jb.SymbolDTO, project: Project) -> None:
         """
         :param symbol_dict: dictionary as returned by the JetBrains plugin client.
         """

src/serena/text_utils.py

@@ -7,6 +7,7 @@ from dataclasses import dataclass, field
 from enum import StrEnum
 from typing import Any, Self
 
+from bs4 import BeautifulSoup
 from joblib import Parallel, delayed
 
 from serena.constants import DEFAULT_SOURCE_FILE_ENCODING
@@ -405,3 +406,21 @@ def search_files(
 
     log.info(f"Found {len(matches)} total matches across {len(filtered_paths)} files")
     return matches
+
+
+def render_html(text: str) -> str:
+    """
+    Remove HTML tags and decode HTML entities from text while preserving the actual content.
+    This keeps type information and structure but removes all formatting.
+
+    :param text: HTML text to clean
+    :return: Plain text without HTML tags and with decoded entities
+    """
+    soup = BeautifulSoup(text, "html.parser")
+    # join text with spaces to avoid concatenation of words
+    text = soup.get_text(separator=" ", strip=True)
+
+    # normalize non-breaking spaces
+    text = text.replace("\xa0", " ")
+
+    return text.strip()

src/serena/tools/jetbrain_types.py

@@ -0,0 +1,50 @@
+from typing import NotRequired, TypedDict
+
+# TODO or not TODO: in principle, we could autogenerate these from the java code. For now not worth the effort.
+
+
+class PositionDTO(TypedDict):
+    line: int
+    col: int
+
+
+class TextRangeDTO(TypedDict):
+    start_pos: PositionDTO
+    end_pos: PositionDTO
+
+
+class SymbolDTO(TypedDict):
+    name_path: str
+    relative_path: str
+    type: str
+    body: str
+    quick_info: NotRequired[str | None]
+    """Quick info text (e.g., type signature) for the symbol, as HTML string."""
+    documentation: NotRequired[str | None]
+    """Documentation text for the symbol (if available), as HTML string."""
+    text_range: TextRangeDTO
+    children: list["SymbolDTO"] | None
+
+
+class SymbolCollectionResponse(TypedDict):
+    symbols: list[SymbolDTO]
+    documentation: NotRequired[str | None]
+    """Docstring of the collection (if applicable - usually present only if the collection is from a single file), 
+    as HTML string."""
+
+
+class TypeHierarchyNode(TypedDict):
+    symbol: SymbolDTO
+    children: list["TypeHierarchyNode"] | None
+
+
+class TypeHierarchy(TypedDict):
+    nodes: list[TypeHierarchyNode]
+    depth: int
+
+
+class TypeHierarchyResponse(TypedDict):
+    symbol: SymbolDTO | None
+    hierarchy: list[TypeHierarchyNode] | None
+    error: str | None
+    num_levels_not_included: int | None

src/serena/tools/jetbrains_plugin_client.py

@@ -5,12 +5,13 @@ Client for the Serena JetBrains Plugin
 import json
 import logging
 from pathlib import Path
-from typing import Any, Optional, Self, TypeVar
+from typing import Any, Optional, Self, TypeVar, cast
 
 import requests
 from requests import Response
 from sensai.util.string import ToStringMixin
 
+import serena.tools.jetbrain_types as jb
 from serena.project import Project
 
 T = TypeVar("T")
@@ -141,21 +142,21 @@ class JetBrainsPluginClient(ToStringMixin):
         name_path: str,
         relative_path: str | None = None,
         include_body: bool = False,
+        include_info: bool = True,
         depth: int = 0,
         include_location: bool = False,
         search_deps: bool = False,
-    ) -> dict[str, Any]:
+    ) -> jb.SymbolCollectionResponse:
         """
         Finds symbols by name.
 
         :param name_path: the name path to match
         :param relative_path: the relative path to which to restrict the search
         :param include_body: whether to include symbol body content
+        :param include_info: whether to include symbol info
         :param depth: depth of children to include (0 = no children)
         :param include_location: whether to include symbol location information
         :param search_deps: whether to also search in dependencies
-
-        :return: Dictionary containing 'symbols' list with matching symbols
         """
         request_data = {
             "namePath": name_path,
@@ -164,27 +165,28 @@ class JetBrainsPluginClient(ToStringMixin):
             "depth": depth,
             "includeLocation": include_location,
             "searchDeps": search_deps,
+            "includeInfo": include_info,
         }
-        return self._make_request("POST", "/findSymbol", request_data)
+        return cast(jb.SymbolCollectionResponse, self._make_request("POST", "/findSymbol", request_data))
 
-    def find_references(self, name_path: str, relative_path: str) -> dict[str, Any]:
+    def find_references(self, name_path: str, relative_path: str, include_info: bool) -> jb.SymbolCollectionResponse:
         """
         Finds references to a symbol.
 
         :param name_path: the name path of the symbol
         :param relative_path: the relative path
-        :return: dictionary containing 'symbols' list with symbol references
+        :param include_info: whether to include symbol info
         """
-        request_data = {"namePath": name_path, "relativePath": relative_path}
-        return self._make_request("POST", "/findReferences", request_data)
+        request_data = {"namePath": name_path, "relativePath": relative_path, "includeInfo": include_info}
+        return cast(jb.SymbolCollectionResponse, self._make_request("POST", "/findReferences", request_data))
 
-    def get_symbols_overview(self, relative_path: str, depth: int) -> dict[str, Any]:
+    def get_symbols_overview(self, relative_path: str, depth: int) -> jb.SymbolCollectionResponse:
         """
         :param relative_path: the relative path to a source file
         :param depth: the depth of children to include (0 = no children)
         """
         request_data = {"relativePath": relative_path, "depth": depth}
-        return self._make_request("POST", "/getSymbolsOverview", request_data)
+        return cast(jb.SymbolCollectionResponse, self._make_request("POST", "/getSymbolsOverview", request_data))
 
     def get_supertypes(
         self,
@@ -192,7 +194,7 @@ class JetBrainsPluginClient(ToStringMixin):
         relative_path: str,
         depth: int | None = None,
         limit_children: int | None = None,
-    ) -> dict[str, Any]:
+    ) -> jb.TypeHierarchyResponse:
         """
         Gets the supertypes (parent classes/interfaces) of a symbol.
 
@@ -200,7 +202,6 @@ class JetBrainsPluginClient(ToStringMixin):
         :param relative_path: the relative path to the file containing the symbol
         :param depth: depth limit for hierarchy traversal (None or 0 for unlimited)
         :param limit_children: optional limit on children per level
-        :return: dictionary containing symbol info and type hierarchy
         """
         request_data = {
             "namePath": name_path,
@@ -208,7 +209,7 @@ class JetBrainsPluginClient(ToStringMixin):
             "depth": depth,
             "limitChildren": limit_children,
         }
-        return self._make_request("POST", "/getSupertypes", request_data)
+        return cast(jb.TypeHierarchyResponse, self._make_request("POST", "/getSupertypes", request_data))
 
     def get_subtypes(
         self,
@@ -216,7 +217,7 @@ class JetBrainsPluginClient(ToStringMixin):
         relative_path: str,
         depth: int | None = None,
         limit_children: int | None = None,
-    ) -> dict[str, Any]:
+    ) -> jb.TypeHierarchyResponse:
         """
         Gets the subtypes (subclasses/implementations) of a symbol.
 
@@ -224,7 +225,6 @@ class JetBrainsPluginClient(ToStringMixin):
         :param relative_path: the relative path to the file containing the symbol
         :param depth: depth limit for hierarchy traversal (None or 0 for unlimited)
         :param limit_children: optional limit on children per level
-        :return: dictionary containing symbol info and type hierarchy
         """
         request_data = {
             "namePath": name_path,
@@ -232,7 +232,7 @@ class JetBrainsPluginClient(ToStringMixin):
             "depth": depth,
             "limitChildren": limit_children,
         }
-        return self._make_request("POST", "/getSubtypes", request_data)
+        return cast(jb.TypeHierarchyResponse, self._make_request("POST", "/getSubtypes", request_data))
 
     def rename_symbol(
         self, name_path: str, relative_path: str, new_name: str, rename_in_comments: bool, rename_in_text_occurrences: bool

src/serena/tools/jetbrains_tools.py

@@ -1,9 +1,14 @@
+import logging
 from collections import defaultdict
-from typing import Any
+from typing import Any, Literal
 
+import serena.tools.jetbrain_types as jb
+from serena.text_utils import render_html
 from serena.tools import Tool, ToolMarkerOptional, ToolMarkerSymbolicRead
 from serena.tools.jetbrains_plugin_client import JetBrainsPluginClient
 
+log = logging.getLogger(__name__)
+
 
 class JetBrainsFindSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional):
     """
@@ -16,6 +21,7 @@ class JetBrainsFindSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional):
         depth: int = 0,
         relative_path: str | None = None,
         include_body: bool = False,
+        include_info: bool = True,
         search_deps: bool = False,
         max_answer_chars: int = -1,
     ) -> str:
@@ -46,6 +52,9 @@ class JetBrainsFindSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional):
             If you have some knowledge about the codebase, you should use this parameter, as it will significantly
             speed up the search as well as reduce the number of results.
         :param include_body: If True, include the symbol's source code. Use judiciously.
+        :param include_info: whether to include additional info (hover-like, typically including docstring and signature),
+            about the symbol (ignored if include_body is True).
+            Default True, info is never included for child symbols and is not included when body is requested.
         :param search_deps: If True, also search in project dependencies (e.g., libraries).
         :param max_answer_chars: max characters for the JSON result. If exceeded, no content is returned.
             -1 means the default value from the config will be used.
@@ -53,14 +62,27 @@ class JetBrainsFindSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional):
         """
         if relative_path == ".":
             relative_path = None
+        if include_body:
+            include_info = False  # we save tokens and don't include info if body is requested
         with JetBrainsPluginClient.from_project(self.project) as client:
             response_dict = client.find_symbol(
                 name_path=name_path_pattern,
                 relative_path=relative_path,
                 depth=depth,
                 include_body=include_body,
+                include_info=include_info,
                 search_deps=search_deps,
             )
+
+            # massage the response to include only 'info' field if requested (instead of both doc and quick info)
+            if not include_body and include_info:
+                symbols = response_dict["symbols"]
+                for symbol in symbols:
+                    documentation = symbol.pop("documentation", None)
+                    quick_info = symbol.pop("quick_info", None)
+                    info = documentation if documentation else quick_info
+                    if info:
+                        symbol["info"] = render_html(info)  # type: ignore
             result = self._to_json(response_dict)
         return self._limit_length(result, max_answer_chars)
 
@@ -74,6 +96,7 @@ class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead, ToolMark
         self,
         name_path: str,
         relative_path: str,
+        include_info: bool = True,
         max_answer_chars: int = -1,
     ) -> str:
         """
@@ -83,6 +106,7 @@ class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead, ToolMark
         :param name_path: name path of the symbol for which to find references; matching logic as described in find symbol tool.
         :param relative_path: the relative path to the file containing the symbol for which to find references.
             Note that here you can't pass a directory but must pass a file.
+        :param include_info: whether to include additional info (hover-like, typically including docstring and signature).
         :param max_answer_chars: max characters for the JSON result. If exceeded, no content is returned. -1 means the
             default value from the config will be used.
         :return: a list of JSON objects with the symbols referencing the requested symbol
@@ -91,6 +115,7 @@ class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead, ToolMark
             response_dict = client.find_references(
                 name_path=name_path,
                 relative_path=relative_path,
+                include_info=include_info,
             )
             result = self._to_json(response_dict)
         return self._limit_length(result, max_answer_chars)
@@ -153,9 +178,12 @@ class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOp
         """
         with JetBrainsPluginClient.from_project(self.project) as client:
             response_dict = client.get_symbols_overview(relative_path=relative_path, depth=depth)
+        documentation = response_dict.pop("documentation", None)
+        if documentation:
+            response_dict["docstring"] = render_html(documentation)  # type: ignore
         symbols = response_dict["symbols"]
-        compact_symbols = self._transform_symbols_to_compact_format(symbols)
-        result = self._to_json(compact_symbols)
+        response_dict["symbols"] = self._transform_symbols_to_compact_format(symbols)  # type: ignore
+        result = self._to_json(response_dict)
         return self._limit_length(result, max_answer_chars)
 
 
@@ -165,7 +193,7 @@ class JetBrainsTypeHierarchyTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptiona
     """
 
     @staticmethod
-    def _transform_hierarchy_nodes(nodes: list[dict] | None) -> dict[str, list]:
+    def _transform_hierarchy_nodes(nodes: list[jb.TypeHierarchyNode] | None) -> dict[str, list]:
         """
         Transform a list of TypeHierarchyNode into a file-grouped compact format.
 
@@ -232,7 +260,6 @@ class JetBrainsTypeHierarchyTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptiona
                     return error_msg
                 if "num_levels_not_included" in supertypes_response:
                     levels_not_included["supertypes"] = supertypes_response["num_levels_not_included"]
-                symbol = supertypes_response["symbol"]
                 supertypes = self._transform_hierarchy_nodes(supertypes_response.get("hierarchy"))
 
             if hierarchy_type in ("sub", "both"):
@@ -243,7 +270,6 @@ class JetBrainsTypeHierarchyTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptiona
                 )
                 if error_msg := subtypes_response.get("error"):
                     return error_msg
-                symbol = subtypes_response["symbol"]
                 if "num_levels_not_included" in subtypes_response:
                     levels_not_included["subtypes"] = subtypes_response["num_levels_not_included"]
                 subtypes = self._transform_hierarchy_nodes(subtypes_response.get("hierarchy"))

uv.lock

@@ -1933,6 +1933,7 @@ version = "0.1.4"
 source = { editable = "." }
 dependencies = [
     { name = "anthropic" },
+    { name = "beautifulsoup4" },
     { name = "docstring-parser" },
     { name = "dotenv" },
     { name = "flask" },
@@ -1988,6 +1989,7 @@ google = [
 requires-dist = [
     { name = "agno", marker = "extra == 'agno'", specifier = ">=2.2.1" },
     { name = "anthropic", specifier = ">=0.54.0" },
+    { name = "beautifulsoup4", specifier = ">=4.14.2" },
     { name = "black", extras = ["jupyter"], marker = "extra == 'dev'", specifier = ">=23.7.0" },
     { name = "docstring-parser", specifier = ">=0.16" },
     { name = "dotenv", specifier = ">=0.9.9" },