From 1d5cf33237309e5f98085fb5337f084019a980ea Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Sat, 3 Jan 2026 18:43:02 +0700 Subject: [PATCH] Support for info in JB symbols and various improvements. Adds bs4 to deps - Added typed dicts for most responses in jetbrains_types.py - Fix bug in code_editor (location key no longer existed, found by mypy after adding the types) Should be merged after next version of plugin release --- pyproject.toml | 3 +- scripts/demo_run_tools.py | 17 +++++-- src/serena/code_editor.py | 2 +- src/serena/symbol.py | 13 ++---- src/serena/text_utils.py | 19 ++++++++ src/serena/tools/jetbrain_types.py | 50 +++++++++++++++++++++ src/serena/tools/jetbrains_plugin_client.py | 34 +++++++------- src/serena/tools/jetbrains_tools.py | 38 +++++++++++++--- uv.lock | 2 + 9 files changed, 139 insertions(+), 39 deletions(-) create mode 100644 src/serena/tools/jetbrain_types.py diff --git a/pyproject.toml b/pyproject.toml index 53bb740d..e83c5d1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ dependencies = [ "tqdm>=4.67.1", "tiktoken>=0.9.0", "anthropic>=0.54.0", + "beautifulsoup4>=4.14.2", ] [[tool.uv.index]] @@ -314,4 +315,4 @@ markers = [ skip = '.git*,*.svg,*.lock,*.min.*' check-hidden = true # ignore-regex = '' -ignore-words-list = 'paket' \ No newline at end of file +ignore-words-list = 'paket' diff --git a/scripts/demo_run_tools.py b/scripts/demo_run_tools.py index 8cbaae79..bad7ed60 100644 --- a/scripts/demo_run_tools.py +++ b/scripts/demo_run_tools.py @@ -7,21 +7,30 @@ import json from pprint import pprint from serena.agent import SerenaAgent +from serena.config.serena_config import SerenaConfig from serena.constants import REPO_ROOT -from serena.tools import FindFileTool, FindReferencingSymbolsTool, GetSymbolsOverviewTool, JetBrainsFindSymbolTool, SearchForPatternTool +from serena.tools import ( + FindFileTool, + FindReferencingSymbolsTool, + JetBrainsFindSymbolTool, + JetBrainsGetSymbolsOverviewTool, + SearchForPatternTool, +) if __name__ == "__main__": - agent = SerenaAgent(project=REPO_ROOT) + serena_config = SerenaConfig.from_config_file() + serena_config.web_dashboard = False + agent = SerenaAgent(project=REPO_ROOT, serena_config=serena_config) # apply a tool find_symbol_tool = agent.get_tool(JetBrainsFindSymbolTool) find_refs_tool = agent.get_tool(FindReferencingSymbolsTool) find_file_tool = agent.get_tool(FindFileTool) search_pattern_tool = agent.get_tool(SearchForPatternTool) - overview_tool = agent.get_tool(GetSymbolsOverviewTool) + overview_tool = agent.get_tool(JetBrainsGetSymbolsOverviewTool) result = agent.execute_task( - lambda: find_symbol_tool.apply("displayBasicStats"), + lambda: find_symbol_tool.apply("SerenaAgent/get_tool_description_override"), ) pprint(json.loads(result)) # input("Press Enter to continue...") diff --git a/src/serena/code_editor.py b/src/serena/code_editor.py index 4f5fe6bb..ac3dca3a 100644 --- a/src/serena/code_editor.py +++ b/src/serena/code_editor.py @@ -422,7 +422,7 @@ class JetBrainsCodeEditor(CodeEditor[JetBrainsSymbol]): if len(symbols) > 1: raise ValueError( f"Found multiple {len(symbols)} symbols with name {name_path} in file {relative_file_path}. " - "Their locations are: \n " + json.dumps([s["location"] for s in symbols], indent=2) + "Their locations are: \n " + json.dumps([s["text_range"] for s in symbols], indent=2) ) return JetBrainsSymbol(symbols[0], self._project) diff --git a/src/serena/symbol.py b/src/serena/symbol.py index da2c7c01..4f71740a 100644 --- a/src/serena/symbol.py +++ b/src/serena/symbol.py @@ -4,10 +4,11 @@ import os from abc import ABC, abstractmethod from collections.abc import Callable, Iterator, Sequence from dataclasses import asdict, dataclass -from typing import TYPE_CHECKING, Any, NotRequired, Self, TypedDict, Union +from typing import TYPE_CHECKING, Any, Self, Union from sensai.util.string import ToStringMixin +import serena.tools.jetbrain_types as jb from solidlsp import SolidLanguageServer from solidlsp.ls import ReferenceInSymbol as LSPReferenceInSymbol from solidlsp.ls_types import Position, SymbolKind, UnifiedSymbolInformation @@ -705,15 +706,7 @@ class LanguageServerSymbolRetriever: class JetBrainsSymbol(Symbol): - class SymbolDict(TypedDict): - name_path: str - relative_path: str - type: str - text_range: NotRequired[dict] - body: NotRequired[str] - children: NotRequired[list["JetBrainsSymbol.SymbolDict"]] - - def __init__(self, symbol_dict: SymbolDict, project: Project) -> None: + def __init__(self, symbol_dict: jb.SymbolDTO, project: Project) -> None: """ :param symbol_dict: dictionary as returned by the JetBrains plugin client. """ diff --git a/src/serena/text_utils.py b/src/serena/text_utils.py index d87e9ad6..3c62e30a 100644 --- a/src/serena/text_utils.py +++ b/src/serena/text_utils.py @@ -7,6 +7,7 @@ from dataclasses import dataclass, field from enum import StrEnum from typing import Any, Self +from bs4 import BeautifulSoup from joblib import Parallel, delayed from serena.constants import DEFAULT_SOURCE_FILE_ENCODING @@ -405,3 +406,21 @@ def search_files( log.info(f"Found {len(matches)} total matches across {len(filtered_paths)} files") return matches + + +def render_html(text: str) -> str: + """ + Remove HTML tags and decode HTML entities from text while preserving the actual content. + This keeps type information and structure but removes all formatting. + + :param text: HTML text to clean + :return: Plain text without HTML tags and with decoded entities + """ + soup = BeautifulSoup(text, "html.parser") + # join text with spaces to avoid concatenation of words + text = soup.get_text(separator=" ", strip=True) + + # normalize non-breaking spaces + text = text.replace("\xa0", " ") + + return text.strip() diff --git a/src/serena/tools/jetbrain_types.py b/src/serena/tools/jetbrain_types.py new file mode 100644 index 00000000..5ec12e00 --- /dev/null +++ b/src/serena/tools/jetbrain_types.py @@ -0,0 +1,50 @@ +from typing import NotRequired, TypedDict + +# TODO or not TODO: in principle, we could autogenerate these from the java code. For now not worth the effort. + + +class PositionDTO(TypedDict): + line: int + col: int + + +class TextRangeDTO(TypedDict): + start_pos: PositionDTO + end_pos: PositionDTO + + +class SymbolDTO(TypedDict): + name_path: str + relative_path: str + type: str + body: str + quick_info: NotRequired[str | None] + """Quick info text (e.g., type signature) for the symbol, as HTML string.""" + documentation: NotRequired[str | None] + """Documentation text for the symbol (if available), as HTML string.""" + text_range: TextRangeDTO + children: list["SymbolDTO"] | None + + +class SymbolCollectionResponse(TypedDict): + symbols: list[SymbolDTO] + documentation: NotRequired[str | None] + """Docstring of the collection (if applicable - usually present only if the collection is from a single file), + as HTML string.""" + + +class TypeHierarchyNode(TypedDict): + symbol: SymbolDTO + children: list["TypeHierarchyNode"] | None + + +class TypeHierarchy(TypedDict): + nodes: list[TypeHierarchyNode] + depth: int + + +class TypeHierarchyResponse(TypedDict): + symbol: SymbolDTO | None + hierarchy: list[TypeHierarchyNode] | None + error: str | None + num_levels_not_included: int | None diff --git a/src/serena/tools/jetbrains_plugin_client.py b/src/serena/tools/jetbrains_plugin_client.py index a2a54cbf..e46ecc8f 100644 --- a/src/serena/tools/jetbrains_plugin_client.py +++ b/src/serena/tools/jetbrains_plugin_client.py @@ -5,12 +5,13 @@ Client for the Serena JetBrains Plugin import json import logging from pathlib import Path -from typing import Any, Optional, Self, TypeVar +from typing import Any, Optional, Self, TypeVar, cast import requests from requests import Response from sensai.util.string import ToStringMixin +import serena.tools.jetbrain_types as jb from serena.project import Project T = TypeVar("T") @@ -141,21 +142,21 @@ class JetBrainsPluginClient(ToStringMixin): name_path: str, relative_path: str | None = None, include_body: bool = False, + include_info: bool = True, depth: int = 0, include_location: bool = False, search_deps: bool = False, - ) -> dict[str, Any]: + ) -> jb.SymbolCollectionResponse: """ Finds symbols by name. :param name_path: the name path to match :param relative_path: the relative path to which to restrict the search :param include_body: whether to include symbol body content + :param include_info: whether to include symbol info :param depth: depth of children to include (0 = no children) :param include_location: whether to include symbol location information :param search_deps: whether to also search in dependencies - - :return: Dictionary containing 'symbols' list with matching symbols """ request_data = { "namePath": name_path, @@ -164,27 +165,28 @@ class JetBrainsPluginClient(ToStringMixin): "depth": depth, "includeLocation": include_location, "searchDeps": search_deps, + "includeInfo": include_info, } - return self._make_request("POST", "/findSymbol", request_data) + return cast(jb.SymbolCollectionResponse, self._make_request("POST", "/findSymbol", request_data)) - def find_references(self, name_path: str, relative_path: str) -> dict[str, Any]: + def find_references(self, name_path: str, relative_path: str, include_info: bool) -> jb.SymbolCollectionResponse: """ Finds references to a symbol. :param name_path: the name path of the symbol :param relative_path: the relative path - :return: dictionary containing 'symbols' list with symbol references + :param include_info: whether to include symbol info """ - request_data = {"namePath": name_path, "relativePath": relative_path} - return self._make_request("POST", "/findReferences", request_data) + request_data = {"namePath": name_path, "relativePath": relative_path, "includeInfo": include_info} + return cast(jb.SymbolCollectionResponse, self._make_request("POST", "/findReferences", request_data)) - def get_symbols_overview(self, relative_path: str, depth: int) -> dict[str, Any]: + def get_symbols_overview(self, relative_path: str, depth: int) -> jb.SymbolCollectionResponse: """ :param relative_path: the relative path to a source file :param depth: the depth of children to include (0 = no children) """ request_data = {"relativePath": relative_path, "depth": depth} - return self._make_request("POST", "/getSymbolsOverview", request_data) + return cast(jb.SymbolCollectionResponse, self._make_request("POST", "/getSymbolsOverview", request_data)) def get_supertypes( self, @@ -192,7 +194,7 @@ class JetBrainsPluginClient(ToStringMixin): relative_path: str, depth: int | None = None, limit_children: int | None = None, - ) -> dict[str, Any]: + ) -> jb.TypeHierarchyResponse: """ Gets the supertypes (parent classes/interfaces) of a symbol. @@ -200,7 +202,6 @@ class JetBrainsPluginClient(ToStringMixin): :param relative_path: the relative path to the file containing the symbol :param depth: depth limit for hierarchy traversal (None or 0 for unlimited) :param limit_children: optional limit on children per level - :return: dictionary containing symbol info and type hierarchy """ request_data = { "namePath": name_path, @@ -208,7 +209,7 @@ class JetBrainsPluginClient(ToStringMixin): "depth": depth, "limitChildren": limit_children, } - return self._make_request("POST", "/getSupertypes", request_data) + return cast(jb.TypeHierarchyResponse, self._make_request("POST", "/getSupertypes", request_data)) def get_subtypes( self, @@ -216,7 +217,7 @@ class JetBrainsPluginClient(ToStringMixin): relative_path: str, depth: int | None = None, limit_children: int | None = None, - ) -> dict[str, Any]: + ) -> jb.TypeHierarchyResponse: """ Gets the subtypes (subclasses/implementations) of a symbol. @@ -224,7 +225,6 @@ class JetBrainsPluginClient(ToStringMixin): :param relative_path: the relative path to the file containing the symbol :param depth: depth limit for hierarchy traversal (None or 0 for unlimited) :param limit_children: optional limit on children per level - :return: dictionary containing symbol info and type hierarchy """ request_data = { "namePath": name_path, @@ -232,7 +232,7 @@ class JetBrainsPluginClient(ToStringMixin): "depth": depth, "limitChildren": limit_children, } - return self._make_request("POST", "/getSubtypes", request_data) + return cast(jb.TypeHierarchyResponse, self._make_request("POST", "/getSubtypes", request_data)) def rename_symbol( self, name_path: str, relative_path: str, new_name: str, rename_in_comments: bool, rename_in_text_occurrences: bool diff --git a/src/serena/tools/jetbrains_tools.py b/src/serena/tools/jetbrains_tools.py index 6f3fd4c5..484bc3a6 100644 --- a/src/serena/tools/jetbrains_tools.py +++ b/src/serena/tools/jetbrains_tools.py @@ -1,9 +1,14 @@ +import logging from collections import defaultdict -from typing import Any +from typing import Any, Literal +import serena.tools.jetbrain_types as jb +from serena.text_utils import render_html from serena.tools import Tool, ToolMarkerOptional, ToolMarkerSymbolicRead from serena.tools.jetbrains_plugin_client import JetBrainsPluginClient +log = logging.getLogger(__name__) + class JetBrainsFindSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): """ @@ -16,6 +21,7 @@ class JetBrainsFindSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): depth: int = 0, relative_path: str | None = None, include_body: bool = False, + include_info: bool = True, search_deps: bool = False, max_answer_chars: int = -1, ) -> str: @@ -46,6 +52,9 @@ class JetBrainsFindSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): If you have some knowledge about the codebase, you should use this parameter, as it will significantly speed up the search as well as reduce the number of results. :param include_body: If True, include the symbol's source code. Use judiciously. + :param include_info: whether to include additional info (hover-like, typically including docstring and signature), + about the symbol (ignored if include_body is True). + Default True, info is never included for child symbols and is not included when body is requested. :param search_deps: If True, also search in project dependencies (e.g., libraries). :param max_answer_chars: max characters for the JSON result. If exceeded, no content is returned. -1 means the default value from the config will be used. @@ -53,14 +62,27 @@ class JetBrainsFindSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): """ if relative_path == ".": relative_path = None + if include_body: + include_info = False # we save tokens and don't include info if body is requested with JetBrainsPluginClient.from_project(self.project) as client: response_dict = client.find_symbol( name_path=name_path_pattern, relative_path=relative_path, depth=depth, include_body=include_body, + include_info=include_info, search_deps=search_deps, ) + + # massage the response to include only 'info' field if requested (instead of both doc and quick info) + if not include_body and include_info: + symbols = response_dict["symbols"] + for symbol in symbols: + documentation = symbol.pop("documentation", None) + quick_info = symbol.pop("quick_info", None) + info = documentation if documentation else quick_info + if info: + symbol["info"] = render_html(info) # type: ignore result = self._to_json(response_dict) return self._limit_length(result, max_answer_chars) @@ -74,6 +96,7 @@ class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead, ToolMark self, name_path: str, relative_path: str, + include_info: bool = True, max_answer_chars: int = -1, ) -> str: """ @@ -83,6 +106,7 @@ class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead, ToolMark :param name_path: name path of the symbol for which to find references; matching logic as described in find symbol tool. :param relative_path: the relative path to the file containing the symbol for which to find references. Note that here you can't pass a directory but must pass a file. + :param include_info: whether to include additional info (hover-like, typically including docstring and signature). :param max_answer_chars: max characters for the JSON result. If exceeded, no content is returned. -1 means the default value from the config will be used. :return: a list of JSON objects with the symbols referencing the requested symbol @@ -91,6 +115,7 @@ class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead, ToolMark response_dict = client.find_references( name_path=name_path, relative_path=relative_path, + include_info=include_info, ) result = self._to_json(response_dict) return self._limit_length(result, max_answer_chars) @@ -153,9 +178,12 @@ class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOp """ with JetBrainsPluginClient.from_project(self.project) as client: response_dict = client.get_symbols_overview(relative_path=relative_path, depth=depth) + documentation = response_dict.pop("documentation", None) + if documentation: + response_dict["docstring"] = render_html(documentation) # type: ignore symbols = response_dict["symbols"] - compact_symbols = self._transform_symbols_to_compact_format(symbols) - result = self._to_json(compact_symbols) + response_dict["symbols"] = self._transform_symbols_to_compact_format(symbols) # type: ignore + result = self._to_json(response_dict) return self._limit_length(result, max_answer_chars) @@ -165,7 +193,7 @@ class JetBrainsTypeHierarchyTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptiona """ @staticmethod - def _transform_hierarchy_nodes(nodes: list[dict] | None) -> dict[str, list]: + def _transform_hierarchy_nodes(nodes: list[jb.TypeHierarchyNode] | None) -> dict[str, list]: """ Transform a list of TypeHierarchyNode into a file-grouped compact format. @@ -232,7 +260,6 @@ class JetBrainsTypeHierarchyTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptiona return error_msg if "num_levels_not_included" in supertypes_response: levels_not_included["supertypes"] = supertypes_response["num_levels_not_included"] - symbol = supertypes_response["symbol"] supertypes = self._transform_hierarchy_nodes(supertypes_response.get("hierarchy")) if hierarchy_type in ("sub", "both"): @@ -243,7 +270,6 @@ class JetBrainsTypeHierarchyTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptiona ) if error_msg := subtypes_response.get("error"): return error_msg - symbol = subtypes_response["symbol"] if "num_levels_not_included" in subtypes_response: levels_not_included["subtypes"] = subtypes_response["num_levels_not_included"] subtypes = self._transform_hierarchy_nodes(subtypes_response.get("hierarchy")) diff --git a/uv.lock b/uv.lock index e8083592..c8bbfb14 100644 --- a/uv.lock +++ b/uv.lock @@ -1933,6 +1933,7 @@ version = "0.1.4" source = { editable = "." } dependencies = [ { name = "anthropic" }, + { name = "beautifulsoup4" }, { name = "docstring-parser" }, { name = "dotenv" }, { name = "flask" }, @@ -1988,6 +1989,7 @@ google = [ requires-dist = [ { name = "agno", marker = "extra == 'agno'", specifier = ">=2.2.1" }, { name = "anthropic", specifier = ">=0.54.0" }, + { name = "beautifulsoup4", specifier = ">=4.14.2" }, { name = "black", extras = ["jupyter"], marker = "extra == 'dev'", specifier = ">=23.7.0" }, { name = "docstring-parser", specifier = ">=0.16" }, { name = "dotenv", specifier = ">=0.9.9" },