From 73f24011f21b51c2a6cb1e792a7234cf7c9ff0c6 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Sat, 20 Dec 2025 22:20:23 +0700 Subject: [PATCH 1/4] More token efficient jetbrains overview tool The grouping by type and the removing of the per-symbol-repeated name_path leads to a huge reduction in output size while the same information is retained! From {"symbols": [{"name_path": "collections", "type": "Py:IMPORT_ELEMENT"}, ... To {"symbols": {"Py:IMPORT_ELEMENT": ["collections", ... --- src/serena/tools/jetbrains_tools.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/serena/tools/jetbrains_tools.py b/src/serena/tools/jetbrains_tools.py index e103854b..0c9f644e 100644 --- a/src/serena/tools/jetbrains_tools.py +++ b/src/serena/tools/jetbrains_tools.py @@ -1,3 +1,5 @@ +from collections import defaultdict + from serena.tools import Tool, ToolMarkerOptional, ToolMarkerSymbolicRead from serena.tools.jetbrains_plugin_client import JetBrainsPluginClient @@ -117,5 +119,15 @@ class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOp response_dict = client.get_symbols_overview( relative_path=relative_path, ) - result = self._to_json(response_dict) + symbols = response_dict["symbols"] + # symbols come with name_path and type, we group them by type for token efficiency + grouped_symbols = defaultdict(list) + for symbol in symbols: + type = symbol.pop("type", "unknown") + if len(symbol) == 1: + # we only have the name_path key, no need to repeat it in each entry + symbol = symbol["name_path"] + grouped_symbols[type].append(symbol) + response_dict["symbols"] = grouped_symbols + result = self._to_json(response_dict) return self._limit_length(result, max_answer_chars) From f0f3d8004e92e653c484c18ac5efe05eb34d280e Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Sun, 21 Dec 2025 10:17:50 +0700 Subject: [PATCH 2/4] Improved output structure in GetSymbolsOverviewTool, major token reduction! As before for jetbrains, grouped and without name_path. Using recursive structure for depth>0 --- src/serena/tools/jetbrains_tools.py | 2 ++ src/serena/tools/symbol_tools.py | 46 +++++++++++++++++++++++++++-- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/src/serena/tools/jetbrains_tools.py b/src/serena/tools/jetbrains_tools.py index 0c9f644e..846ccb6f 100644 --- a/src/serena/tools/jetbrains_tools.py +++ b/src/serena/tools/jetbrains_tools.py @@ -98,6 +98,8 @@ class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOp Retrieves an overview of the top-level symbols within a specified file using the JetBrains backend """ + # TODO: support depth param, structure output in the same way as in GetSymbolsOverviewTool (by extracting + # the post-processing logic used there into a shared utility function) def apply( self, relative_path: str, diff --git a/src/serena/tools/symbol_tools.py b/src/serena/tools/symbol_tools.py index 75d0e5c9..64261874 100644 --- a/src/serena/tools/symbol_tools.py +++ b/src/serena/tools/symbol_tools.py @@ -7,6 +7,8 @@ from collections.abc import Sequence from copy import copy from typing import Any +from black.trans import defaultdict + from serena.tools import ( SUCCESS_RESULT, Tool, @@ -61,7 +63,15 @@ class GetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead): :param max_answer_chars: if the overview is longer than this number of characters, no content will be returned. -1 means the default value from the config will be used. Don't adjust unless there is really no other way to get the content required for the task. - :return: a JSON object containing info about top-level symbols in the file + :return: a JSON object containing symbols grouped by kind in a compact format. + + For depth=0, the structure is: {"Kind1": ["name1", "name2"], "Kind2": ["name3"]} + For depth>0, symbols with children use nested dicts: {"Class": [{"ClassName": {"Method": ["method1", "method2"]}}]} + + The name_path for any symbol can be inferred from the structure: + - Top-level symbols: name_path = name + - Nested symbols: name_path = parent_name + "/" + name + Example: "method1" under class "ClassName" has name_path "ClassName/method1" """ symbol_retriever = self.create_language_server_symbol_retriever() file_path = os.path.join(self.project.project_root, relative_path) @@ -73,9 +83,41 @@ class GetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead): if os.path.isdir(file_path): raise ValueError(f"Expected a file path, but got a directory path: {relative_path}. ") result = symbol_retriever.get_symbol_overview(relative_path, depth=depth)[relative_path] - result_json_str = self._to_json(result) + # Transform to compact format + compact_result = self._transform_symbols_to_compact_format(result) + result_json_str = self._to_json(compact_result) return self._limit_length(result_json_str, max_answer_chars) + @staticmethod + def _transform_symbols_to_compact_format(symbols: list[dict[str, Any]]) -> dict[str, list]: + """ + Transform symbol overview from verbose format to compact grouped format. + + Groups symbols by kind and uses names instead of full symbol objects. + For symbols with children, creates nested dictionaries. + + The name_path can be inferred from the hierarchical structure: + - Top-level symbols: name_path = name + - Nested symbols: name_path = parent_name + "/" + name + For example, "convert" under class "ProjectType" has name_path "ProjectType/convert". + """ + result = defaultdict(list) + + for symbol in symbols: + kind = symbol.get("kind", "Unknown") + name = symbol.get("name", "unknown") + children = symbol.get("children", []) + + if children: + # Symbol has children: create nested dict {name: children_dict} + children_dict = GetSymbolsOverviewTool._transform_symbols_to_compact_format(children) + result[kind].append({name: children_dict}) + else: + # Symbol has no children: just add the name + result[kind].append(name) + + return result + class FindSymbolTool(Tool, ToolMarkerSymbolicRead): """ From 025ef322206a35db98e5bb671c8fda2c5511b0db Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Mon, 22 Dec 2025 13:25:40 +0700 Subject: [PATCH 3/4] Improvements in JetBrains overview tool Compact version and support for depth>0 and filtering out imports. Backwards compatible (previous plugin versions won't crash), but users have to update to 2023.2.3 to benefit from the change --- src/serena/tools/jetbrains_plugin_client.py | 6 ++- src/serena/tools/jetbrains_tools.py | 52 +++++++++++++++------ 2 files changed, 41 insertions(+), 17 deletions(-) diff --git a/src/serena/tools/jetbrains_plugin_client.py b/src/serena/tools/jetbrains_plugin_client.py index cd31b7b1..a02d4600 100644 --- a/src/serena/tools/jetbrains_plugin_client.py +++ b/src/serena/tools/jetbrains_plugin_client.py @@ -174,11 +174,13 @@ class JetBrainsPluginClient(ToStringMixin): request_data = {"namePath": name_path, "relativePath": relative_path} return self._make_request("POST", "/findReferences", request_data) - def get_symbols_overview(self, relative_path: str) -> dict[str, Any]: + def get_symbols_overview(self, relative_path: str, depth: int, exclude_imports: bool = True) -> dict[str, Any]: """ :param relative_path: the relative path to a source file + :param depth: the depth of children to include (0 = no children) + :param exclude_imports: whether to exclude import statements from the overview """ - request_data = {"relativePath": relative_path} + request_data = {"relativePath": relative_path, "depth": depth, "excludeImports": exclude_imports} return self._make_request("POST", "/getSymbolsOverview", request_data) def rename_symbol( diff --git a/src/serena/tools/jetbrains_tools.py b/src/serena/tools/jetbrains_tools.py index 846ccb6f..afe3113e 100644 --- a/src/serena/tools/jetbrains_tools.py +++ b/src/serena/tools/jetbrains_tools.py @@ -1,4 +1,5 @@ from collections import defaultdict +from typing import Any from serena.tools import Tool, ToolMarkerOptional, ToolMarkerSymbolicRead from serena.tools.jetbrains_plugin_client import JetBrainsPluginClient @@ -98,11 +99,41 @@ class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOp Retrieves an overview of the top-level symbols within a specified file using the JetBrains backend """ - # TODO: support depth param, structure output in the same way as in GetSymbolsOverviewTool (by extracting - # the post-processing logic used there into a shared utility function) + @staticmethod + def _transform_symbols_to_compact_format(symbols: list[dict[str, Any]]) -> dict[str, list]: + """ + Transform symbol overview from verbose format to compact grouped format. + + Groups symbols by kind and uses names instead of full symbol objects. + For symbols with children, creates nested dictionaries. + + The name_path can be inferred from the hierarchical structure: + - Top-level symbols: name_path = name + - Nested symbols: name_path = parent_name + "/" + name + For example, "convert" under class "ProjectType" has name_path "ProjectType/convert". + """ + result = defaultdict(list) + + for symbol in symbols: + kind = symbol.get("type", "Unknown") + name_path = symbol["name_path"] + name = name_path.split("/")[-1] + children = symbol.get("children", []) + + if children: + # Symbol has children: create nested dict {name: children_dict} + children_dict = JetBrainsGetSymbolsOverviewTool._transform_symbols_to_compact_format(children) + result[kind].append({name: children_dict}) + else: + # Symbol has no children: just add the name + result[kind].append(name) + + return result + def apply( self, relative_path: str, + depth: int = 0, max_answer_chars: int = -1, ) -> str: """ @@ -113,23 +144,14 @@ class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOp or by using the `list_dir` and `find_file` tools (or similar). :param relative_path: the relative path to the file to get the overview of + :param depth: depth up to which descendants shall be retrieved (e.g., use 1 to also retrieve immediate children). :param max_answer_chars: max characters for the JSON result. If exceeded, no content is returned. -1 means the default value from the config will be used. :return: a JSON object containing the symbols """ with JetBrainsPluginClient.from_project(self.project) as client: - response_dict = client.get_symbols_overview( - relative_path=relative_path, - ) + response_dict = client.get_symbols_overview(relative_path=relative_path, depth=depth) symbols = response_dict["symbols"] - # symbols come with name_path and type, we group them by type for token efficiency - grouped_symbols = defaultdict(list) - for symbol in symbols: - type = symbol.pop("type", "unknown") - if len(symbol) == 1: - # we only have the name_path key, no need to repeat it in each entry - symbol = symbol["name_path"] - grouped_symbols[type].append(symbol) - response_dict["symbols"] = grouped_symbols - result = self._to_json(response_dict) + compact_symbols = self._transform_symbols_to_compact_format(symbols) + result = self._to_json(compact_symbols) return self._limit_length(result, max_answer_chars) From 5801982113bd4434baf5bb84e928c3c82485db20 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Mon, 22 Dec 2025 13:29:02 +0700 Subject: [PATCH 4/4] Docstrings, minor fixes [ci skip] --- src/serena/tools/jetbrains_plugin_client.py | 5 ++--- src/serena/tools/jetbrains_tools.py | 2 +- src/serena/tools/symbol_tools.py | 11 +---------- 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/src/serena/tools/jetbrains_plugin_client.py b/src/serena/tools/jetbrains_plugin_client.py index a02d4600..7a3c6f38 100644 --- a/src/serena/tools/jetbrains_plugin_client.py +++ b/src/serena/tools/jetbrains_plugin_client.py @@ -174,13 +174,12 @@ class JetBrainsPluginClient(ToStringMixin): request_data = {"namePath": name_path, "relativePath": relative_path} return self._make_request("POST", "/findReferences", request_data) - def get_symbols_overview(self, relative_path: str, depth: int, exclude_imports: bool = True) -> dict[str, Any]: + def get_symbols_overview(self, relative_path: str, depth: int) -> dict[str, Any]: """ :param relative_path: the relative path to a source file :param depth: the depth of children to include (0 = no children) - :param exclude_imports: whether to exclude import statements from the overview """ - request_data = {"relativePath": relative_path, "depth": depth, "excludeImports": exclude_imports} + request_data = {"relativePath": relative_path, "depth": depth} return self._make_request("POST", "/getSymbolsOverview", request_data) def rename_symbol( diff --git a/src/serena/tools/jetbrains_tools.py b/src/serena/tools/jetbrains_tools.py index afe3113e..a14a9402 100644 --- a/src/serena/tools/jetbrains_tools.py +++ b/src/serena/tools/jetbrains_tools.py @@ -147,7 +147,7 @@ class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOp :param depth: depth up to which descendants shall be retrieved (e.g., use 1 to also retrieve immediate children). :param max_answer_chars: max characters for the JSON result. If exceeded, no content is returned. -1 means the default value from the config will be used. - :return: a JSON object containing the symbols + :return: a JSON object containing the symbols grouped by kind in a compact format. """ with JetBrainsPluginClient.from_project(self.project) as client: response_dict = client.get_symbols_overview(relative_path=relative_path, depth=depth) diff --git a/src/serena/tools/symbol_tools.py b/src/serena/tools/symbol_tools.py index 64261874..bcd244a5 100644 --- a/src/serena/tools/symbol_tools.py +++ b/src/serena/tools/symbol_tools.py @@ -3,12 +3,11 @@ Language server-related tools """ import os +from collections import defaultdict from collections.abc import Sequence from copy import copy from typing import Any -from black.trans import defaultdict - from serena.tools import ( SUCCESS_RESULT, Tool, @@ -64,14 +63,6 @@ class GetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead): no content will be returned. -1 means the default value from the config will be used. Don't adjust unless there is really no other way to get the content required for the task. :return: a JSON object containing symbols grouped by kind in a compact format. - - For depth=0, the structure is: {"Kind1": ["name1", "name2"], "Kind2": ["name3"]} - For depth>0, symbols with children use nested dicts: {"Class": [{"ClassName": {"Method": ["method1", "method2"]}}]} - - The name_path for any symbol can be inferred from the structure: - - Top-level symbols: name_path = name - - Nested symbols: name_path = parent_name + "/" + name - Example: "method1" under class "ClassName" has name_path "ClassName/method1" """ symbol_retriever = self.create_language_server_symbol_retriever() file_path = os.path.join(self.project.project_root, relative_path)