Major: replace location-based symbol referencing by name_path based

Also:
1. find_references now cannot return bodies, instead content around the reference is included
2. find_reference_snippets tool removed since it's no longer necessary

scripts/demo_run_tools.py

@@ -3,10 +3,10 @@ This script demonstrates how to use Serena's tools locally, useful
 for testing or development. Here the tools will be operation the serena repo itself.
 """
 
-import json
 from pprint import pprint
 
 from serena.agent import *
+from serena.constants import REPO_ROOT
 
 
 @dataclass
@@ -20,10 +20,10 @@ class InMemorySerenaConfig(SerenaConfigBase):
 
 
 if __name__ == "__main__":
-    project_path = str(Path("test") / "resources" / "repos" / "python" / "test_repo")
-    agent = SerenaAgent(project=project_path, serena_config=InMemorySerenaConfig())
+    # project_path = str(Path("test") / "resources" / "repos" / "python" / "test_repo")
+    agent = SerenaAgent(project=REPO_ROOT)
 
     # apply a tool
-    find_symbol_tool = agent.get_tool(FindSymbolTool)
-    print("Finding the symbol 'VariableContainer'\n")
-    pprint(json.loads(find_symbol_tool.apply("VariableContainer", within_relative_path=str(Path("test_repo") / "variables.py"))))
+    find_refs_tool = agent.get_tool(FindReferencingSymbolsTool)
+    print("Finding the symbol 'SyncLanguageServer'\n")
+    pprint(json.loads(find_refs_tool.apply(name_path="SyncLanguageServer", relative_file_path="src/multilspy/language_server.py")))

src/serena/agent.py

@@ -37,7 +37,7 @@ from serena.config import SerenaAgentContext, SerenaAgentMode
 from serena.constants import PROJECT_TEMPLATE_FILE, SERENA_MANAGED_DIR_NAME
 from serena.dashboard import MemoryLogHandler, SerenaDashboardAPI
 from serena.prompt_factory import PromptFactory, SerenaPromptFactory
-from serena.symbol import SymbolLocation, SymbolManager
+from serena.symbol import SymbolManager
 from serena.text_utils import search_files
 from serena.util.file_system import scan_directory
 from serena.util.general import load_yaml, save_yaml
@@ -1196,98 +1196,50 @@ class FindReferencingSymbolsTool(Tool):
 
     def apply(
         self,
-        relative_path: str,
-        line: int,
-        column: int,
-        include_body: bool = False,
+        name_path: str,
+        relative_file_path: str,
         include_kinds: list[int] | None = None,
         exclude_kinds: list[int] | None = None,
         max_answer_chars: int = _DEFAULT_MAX_ANSWER_LENGTH,
     ) -> str:
         """
-        Finds symbols that reference the symbol at the given location.
+        Finds symbols that reference the symbol at the given `name_path`. The result will contain metadata about the referencing symbols
+        as well as a short code snippet around the reference (unless `include_body` is True, then the short snippet will be omitted).
         Note that among other kinds of references, this function can be used to find (direct) subclasses of a class,
         as subclasses are referencing symbols that have the kind class.
 
-        :param relative_path: the relative path to the file containing the symbol
-        :param line: the line number
-        :param column: the column
-        :param include_body: whether to include the body of the symbols in the result.
-            Note that this might lead to a very long output, so you should only use this if you actually need the body
-            of the referencing symbols for the task at hand. Usually it is a better idea to find
-            the referencing symbols without the body and then use the find_symbol tool to get the body of
-            specific symbols if needed.
-        :param include_kinds: an optional list of integers representing the LSP symbol kinds to include.
-            If provided, only symbols of the given kinds will be included in the result.
-            Valid kinds:
-            1=file, 2=module, 3=namespace, 4=package, 5=class, 6=method, 7=property, 8=field, 9=constructor, 10=enum,
-            11=interface, 12=function, 13=variable, 14=constant, 15=string, 16=number, 17=boolean, 18=array, 19=object,
-            20=key, 21=null, 22=enum member, 23=struct, 24=event, 25=operator, 26=type parameter
-        :param exclude_kinds: If provided, symbols of the given kinds will be excluded from the result.
-            Takes precedence over include_kinds.
-        :param max_answer_chars: if the output is longer than this number of characters,
-            no content will be returned. Don't adjust unless there is really no other way to get the content
-            required for the task. Instead, if the output is too long, you should
-            make a stricter query.
+        :param name_path: for finding the symbol to find references for, same logic as in the `find_symbol` tool.
+        :param relative_file_path: the relative path to the file containing the symbol for which to find references.
+        :param include_kinds: same as in the `find_symbol` tool.
+        :param exclude_kinds: same as in the `find_symbol` tool.
+        :param max_answer_chars: same as in the `find_symbol` tool.
         :return: a list of JSON objects with the symbols referencing the requested symbol
         """
+        include_body = False  # It is probably never a good idea to include the body of the referencing symbols
         parsed_include_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in include_kinds] if include_kinds else None
         parsed_exclude_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in exclude_kinds] if exclude_kinds else None
-        symbols = self.symbol_manager.find_referencing_symbols_by_location(
-            SymbolLocation(relative_path, line, column),
+        references_in_symbols = self.symbol_manager.find_referencing_symbols(
+            name_path,
+            relative_file_path=relative_file_path,
             include_body=include_body,
             include_kinds=parsed_include_kinds,
             exclude_kinds=parsed_exclude_kinds,
         )
-        symbol_dicts = [s.to_dict(kind=True, location=True, depth=0, include_body=include_body) for s in symbols]
-        result = json.dumps(symbol_dicts)
+        reference_dicts = []
+        for ref in references_in_symbols:
+            ref_dict = ref.symbol.to_dict(kind=True, location=True, depth=0, include_body=include_body)
+            if not include_body:
+                ref_relative_path = ref.symbol.location.relative_path
+                assert ref_relative_path is not None, f"Referencing symbol {ref.symbol.name} has no relative path, this is likely a bug."
+                content_around_ref = self.language_server.retrieve_content_around_line(
+                    relative_file_path=ref_relative_path, line=ref.line, context_lines_before=1, context_lines_after=1
+                )
+                ref_dict["content_around_reference"] = content_around_ref.to_display_string()
+            reference_dicts.append(ref_dict)
+        result = json.dumps(reference_dicts)
         return self._limit_length(result, max_answer_chars)
 
 
-class FindReferencingCodeSnippetsTool(Tool):
-    """
-    Finds code snippets in which the symbol at the given location is referenced.
-    """
-
-    def apply(
-        self,
-        relative_path: str,
-        line: int,
-        column: int,
-        context_lines_before: int = 0,
-        context_lines_after: int = 0,
-        max_answer_chars: int = _DEFAULT_MAX_ANSWER_LENGTH,
-    ) -> str:
-        """
-        Returns short code snippets where the symbol at the given location is referenced.
-
-        Contrary to the `find_referencing_symbols` tool, this tool returns references that are not symbols but instead
-        code snippets that may or may not be contained in a symbol (for example, file-level calls).
-        It may make sense to use this tool to get a quick overview of the code that references
-        the symbol. Usually, just looking at code snippets is not enough to understand the full context,
-        unless the case you are investigating is very simple,
-        or you already have read the relevant symbols using the find_referencing_symbols tool and
-        now want to get an overview of how the referenced symbol (at the given location) is used in them.
-        The size of the snippets is controlled by the context_lines_before and context_lines_after parameters.
-
-        :param relative_path: the relative path to the file containing the symbol
-        :param line: the line number of the symbol to find references for
-        :param column: the column of the symbol to find references for
-        :param context_lines_before: the number of lines to include before the line containing the reference
-        :param context_lines_after: the number of lines to include after the line containing the reference
-        :param max_answer_chars: if the output is longer than this number of characters,
-            no content will be returned. Don't adjust unless there is really no other way to get the content
-            required for the task. Instead, if the output is too long, you should
-            make a stricter query.
-        """
-        matches = self.language_server.request_references_with_content(
-            relative_path, line, column, context_lines_before, context_lines_after
-        )
-        result = [match.to_display_string() for match in matches]
-        result_json_str = json.dumps(result)
-        return self._limit_length(result_json_str, max_answer_chars)
-
-
 class ReplaceSymbolBodyTool(Tool, ToolMarkerCanEdit):
     """
     Replaces the full definition of a symbol.
@@ -1295,23 +1247,21 @@ class ReplaceSymbolBodyTool(Tool, ToolMarkerCanEdit):
 
     def apply(
         self,
+        name_path: str,
         relative_path: str,
-        line: int,
-        column: int,
         body: str,
     ) -> str:
         """
-        Replaces the body of the symbol at the given location.
-        Important: Do not try to guess symbol locations but instead use the find_symbol tool to get the correct location.
+        Replaces the body of the symbol with the given `name_path`.
 
+        :param name_path: for finding the symbol to replace, same logic as in the `find_symbol` tool.
         :param relative_path: the relative path to the file containing the symbol
-        :param line: the line number
-        :param column: the column
         :param body: the new symbol body. Important: Provide the correct level of indentation
             (as the original body). Note that the first line must not be indented (i.e. no leading spaces).
         """
-        self.symbol_manager.replace_body_at_location(
-            SymbolLocation(relative_path, line, column),
+        self.symbol_manager.replace_body(
+            name_path,
+            relative_file_path=relative_path,
             body=body,
         )
         return SUCCESS_RESULT
@@ -1324,23 +1274,22 @@ class InsertAfterSymbolTool(Tool, ToolMarkerCanEdit):
 
     def apply(
         self,
+        name_path: str,
         relative_path: str,
-        line: int,
-        column: int,
         body: str,
     ) -> str:
         """
         Inserts the given body/content after the end of the definition of the given symbol (via the symbol's location).
         A typical use case is to insert a new class, function, method, field or variable assignment.
 
+        :param name_path: for finding the symbol to insert after, same logic as in the `find_symbol` tool.
         :param relative_path: the relative path to the file containing the symbol
-        :param line: the line number
-        :param column: the column
-        :param body: the body/content to be inserted
+        :param body: the body/content to be inserted. Important: the insterted code will automatically have the
+            same indentation as the symbol's body, so you do not need to provide any indentation.
         """
-        location = SymbolLocation(relative_path, line, column)
-        self.symbol_manager.insert_after_symbol_at_location(
-            location,
+        self.symbol_manager.insert_after_symbol(
+            name_path,
+            relative_file_path=relative_path,
             body=body,
         )
         return SUCCESS_RESULT
@@ -1353,9 +1302,8 @@ class InsertBeforeSymbolTool(Tool, ToolMarkerCanEdit):
 
     def apply(
         self,
+        name_path: str,
         relative_path: str,
-        line: int,
-        column: int,
         body: str,
     ) -> str:
         """
@@ -1363,13 +1311,14 @@ class InsertBeforeSymbolTool(Tool, ToolMarkerCanEdit):
         A typical use case is to insert a new class, function, method, field or variable assignment.
         It also can be used to insert a new import statement before the first symbol in the file.
 
+        :param name_path: for finding the symbol to insert before, same logic as in the `find_symbol` tool.
         :param relative_path: the relative path to the file containing the symbol
-        :param line: the line number
-        :param column: the column
-        :param body: the body/content to be inserted
+        :param body: the body/content to be inserted. Important: the insterted code will automatically have the
+            same indentation as the symbol's body, so you do not need to provide any indentation.
         """
-        self.symbol_manager.insert_before_symbol_at_location(
-            SymbolLocation(relative_path, line, column),
+        self.symbol_manager.insert_before_symbol(
+            name_path,
+            relative_file_path=relative_path,
             body=body,
         )
         return SUCCESS_RESULT

src/serena/symbol.py

@@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Literal, NamedTuple, Self, Union
 from sensai.util.string import ToStringMixin
 
 from multilspy import SyncLanguageServer
+from multilspy.language_server import ReferenceInSymbol as LSPReferenceInSymbol
 from multilspy.multilspy_types import Position, SymbolKind, UnifiedSymbolInformation
 
 if TYPE_CHECKING:
@@ -468,6 +469,24 @@ class Symbol(ToStringMixin):
         return result
 
 
+@dataclass
+class ReferenceInSymbol(ToStringMixin):
+    """Same as the class of the same name in the language server, but using Serena's Symbol class.
+    Be careful to not confuse it with counterpart!
+    """
+
+    symbol: Symbol
+    line: int
+    character: int
+
+    def get_relative_path(self) -> str | None:
+        return self.symbol.location.relative_path
+
+    @classmethod
+    def from_lsp_reference(cls, reference: LSPReferenceInSymbol) -> Self:
+        return cls(symbol=Symbol(reference.symbol), line=reference.line, character=reference.character)
+
+
 class SymbolManager:
     def __init__(self, lang_server: SyncLanguageServer, agent: Union["SerenaAgent", None] = None) -> None:
         """
@@ -478,9 +497,6 @@ class SymbolManager:
         self.lang_server = lang_server
         self.agent = agent
 
-    def _to_symbols(self, items: list[UnifiedSymbolInformation]) -> list[Symbol]:
-        return [Symbol(s) for s in items]
-
     def find_by_name(
         self,
         name_path: str,
@@ -527,27 +543,33 @@ class SymbolManager:
         include_body: bool = False,
         include_kinds: Sequence[SymbolKind] | None = None,
         exclude_kinds: Sequence[SymbolKind] | None = None,
-    ) -> tuple[Symbol, list[Symbol]] | None:
+    ) -> list[ReferenceInSymbol]:
         """
         Find all symbols that reference the symbol with the given name.
         If multiple symbols fit the name (e.g. for variables that are overwritten), will use the first one.
+
+        :param name_path: the name path of the symbol to find
+        :param relative_file_path: the relative path of the file in which the referenced symbol is defined.
+        :param include_body: whether to include the body of all symbols in the result.
+            Not recommended, as the referencing symbols will often be files, and thus the bodies will be very long.
+        :param include_kinds: which kinds of symbols to include in the result.
+        :param exclude_kinds: which kinds of symbols to exclude from the result.
         """
         symbol_candidates = self.find_by_name(name_path, substring_matching=False, within_relative_path=relative_file_path)
         if len(symbol_candidates) == 0:
             log.warning(f"No symbol with name {name_path} found in file {relative_file_path}")
-            return None
+            return []
         if len(symbol_candidates) > 1:
             log.error(
                 f"Found {len(symbol_candidates)} symbols with name {name_path} in file {relative_file_path}."
                 f"May be an overwritten variable, in which case you can ignore this error. Proceeding with the first one. "
                 f"Found symbols for {name_path=} in {relative_file_path=}: \n"
-                f""
+                f"{json.dumps([s.location.to_dict() for s in symbol_candidates], indent=2)}"
             )
         symbol = symbol_candidates[0]
-        referencing_symbols = self.find_referencing_symbols_by_location(
+        return self.find_referencing_symbols_by_location(
             symbol.location, include_body=include_body, include_kinds=include_kinds, exclude_kinds=exclude_kinds
         )
-        return symbol, referencing_symbols
 
     def find_referencing_symbols_by_location(
         self,
@@ -555,13 +577,14 @@ class SymbolManager:
         include_body: bool = False,
         include_kinds: Sequence[SymbolKind] | None = None,
         exclude_kinds: Sequence[SymbolKind] | None = None,
-    ) -> list[Symbol]:
+    ) -> list[ReferenceInSymbol]:
         """
         Find all symbols that reference the symbol at the given location.
 
         :param symbol_location: the location of the symbol for which to find references.
             Does not need to include an end_line, as it is unused in the search.
         :param include_body: whether to include the body of all symbols in the result.
+            Not recommended, as the referencing symbols will often be files, and thus the bodies will be very long.
             Note: you can filter out the bodies of the children if you set include_children_body=False
             in the to_dict method.
         :param include_kinds: an optional sequence of ints representing the LSP symbol kind.
@@ -575,22 +598,23 @@ class SymbolManager:
         assert symbol_location.relative_path is not None
         assert symbol_location.line is not None
         assert symbol_location.column is not None
-        symbol_dicts = self.lang_server.request_referencing_symbols(
+        references = self.lang_server.request_referencing_symbols(
             relative_file_path=symbol_location.relative_path,
             line=symbol_location.line,
             column=symbol_location.column,
             include_imports=False,
             include_self=False,
             include_body=include_body,
+            include_file_symbols=True,
         )
 
         if include_kinds is not None:
-            symbol_dicts = [s for s in symbol_dicts if s["kind"] in include_kinds]
+            references = [s for s in references if s.symbol["kind"] in include_kinds]
 
         if exclude_kinds is not None:
-            symbol_dicts = [s for s in symbol_dicts if s["kind"] not in exclude_kinds]
+            references = [s for s in references if s.symbol["kind"] not in exclude_kinds]
 
-        return self._to_symbols(symbol_dicts)
+        return [ReferenceInSymbol.from_lsp_reference(r) for r in references]
 
     @contextmanager
     def _edited_file(self, relative_path: str) -> Iterator[None]:

src/serena/text_utils.py

@@ -3,7 +3,7 @@ import re
 from collections.abc import Callable
 from dataclasses import dataclass, field
 from enum import StrEnum
-from typing import Any
+from typing import Any, Self
 
 from joblib import Parallel, delayed
 from pathspec import PathSpec
@@ -38,11 +38,16 @@ class TextLine:
             return "  >"
         return "..."
 
-    def format_line(self) -> str:
-        """Format the line for display with line number and content."""
+    def format_line(self, include_line_numbers: bool = True) -> str:
+        """Format the line for display (e.g.,for logging or passing to an LLM).
+
+        :param include_line_numbers: Whether to include the line number in the result.
+        """
         prefix = self.get_display_prefix()
-        line_num = str(self.line_number).rjust(4)
-        return f"{prefix}{line_num}: {self.line_content}"
+        if include_line_numbers:
+            line_num = str(self.line_number).rjust(4)
+            prefix = f"{prefix}{line_num}"
+        return f"{prefix}:{self.line_content}"
 
 
 @dataclass(kw_only=True)
@@ -52,7 +57,7 @@ class MatchedConsecutiveLines:
     """
 
     lines: list[TextLine]
-    """All lines in the context of the match. At least one of them should be of match_type MATCH."""
+    """All lines in the context of the match. At least one of them is of `match_type` `MATCH`."""
     source_file_path: str | None = None
     """Path to the file where the match was found (Metadata)."""
 
@@ -84,8 +89,27 @@ class MatchedConsecutiveLines:
     def num_matched_lines(self) -> int:
         return len(self.matched_lines)
 
-    def to_display_string(self) -> str:
-        return "\n".join([line.format_line() for line in self.lines])
+    def to_display_string(self, include_line_numbers: bool = True) -> str:
+        return "\n".join([line.format_line(include_line_numbers) for line in self.lines])
+
+    @classmethod
+    def from_file_contents(
+        cls, file_contents: str, line: int, context_lines_before: int = 0, context_lines_after: int = 0, source_file_path: str | None = None
+    ) -> Self:
+        line_contents = file_contents.split("\n")
+        start_lineno = max(0, line - context_lines_before)
+        end_lineno = min(len(line_contents) - 1, line + context_lines_after)
+        text_lines: list[TextLine] = []
+        # before the line
+        for lineno in range(start_lineno, line):
+            text_lines.append(TextLine(line_number=lineno, line_content=line_contents[lineno], match_type=LineType.BEFORE_MATCH))
+        # the line
+        text_lines.append(TextLine(line_number=line, line_content=line_contents[line], match_type=LineType.MATCH))
+        # after the line
+        for lineno in range(line + 1, end_lineno + 1):
+            text_lines.append(TextLine(line_number=lineno, line_content=line_contents[lineno], match_type=LineType.AFTER_MATCH))
+
+        return cls(lines=text_lines, source_file_path=source_file_path)
 
 
 def search_text(

test/multilspy/python/test_symbol_retrieval.py

@@ -40,7 +40,7 @@ class TestLanguageServerSymbols:
         """Test request_referencing_symbols for a variable."""
         file_path = os.path.join("test_repo", "variables.py")
         # Line 75 contains the field status that is later modified
-        ref_symbols = language_server.request_referencing_symbols(file_path, 74, 4)
+        ref_symbols = [ref.symbol for ref in language_server.request_referencing_symbols(file_path, 74, 4)]
 
         assert len(ref_symbols) > 0
         ref_lines = [ref["location"]["range"]["start"]["line"] for ref in ref_symbols if "location" in ref and "range" in ref["location"]]
@@ -111,7 +111,9 @@ class TestLanguageServerSymbols:
         if not create_user_symbol or "selectionRange" not in create_user_symbol:
             raise AssertionError("create_user symbol or its selectionRange not found")
         sel_start = create_user_symbol["selectionRange"]["start"]
-        ref_symbols = language_server.request_referencing_symbols(file_path, sel_start["line"], sel_start["character"])
+        ref_symbols = [
+            ref.symbol for ref in language_server.request_referencing_symbols(file_path, sel_start["line"], sel_start["character"])
+        ]
         assert len(ref_symbols) > 0, "No referencing symbols found for create_user (selectionRange)"
 
         # Verify the structure of referencing symbols
@@ -133,7 +135,9 @@ class TestLanguageServerSymbols:
         if not user_symbol or "selectionRange" not in user_symbol:
             raise AssertionError("User symbol or its selectionRange not found")
         sel_start = user_symbol["selectionRange"]["start"]
-        ref_symbols = language_server.request_referencing_symbols(file_path, sel_start["line"], sel_start["character"])
+        ref_symbols = [
+            ref.symbol for ref in language_server.request_referencing_symbols(file_path, sel_start["line"], sel_start["character"])
+        ]
         services_references = [
             symbol
             for symbol in ref_symbols
@@ -152,7 +156,9 @@ class TestLanguageServerSymbols:
         if not get_user_symbol or "selectionRange" not in get_user_symbol:
             raise AssertionError("get_user symbol or its selectionRange not found")
         sel_start = get_user_symbol["selectionRange"]["start"]
-        ref_symbols = language_server.request_referencing_symbols(file_path, sel_start["line"], sel_start["character"])
+        ref_symbols = [
+            ref.symbol for ref in language_server.request_referencing_symbols(file_path, sel_start["line"], sel_start["character"])
+        ]
         method_refs = [
             symbol
             for symbol in ref_symbols
@@ -169,7 +175,7 @@ class TestLanguageServerSymbols:
         file_path = os.path.join("test_repo", "services.py")
         # Line 3 is a blank line or comment
         try:
-            ref_symbols = language_server.request_referencing_symbols(file_path, 3, 0)
+            ref_symbols = [ref.symbol for ref in language_server.request_referencing_symbols(file_path, 3, 0)]
             # If we get here, make sure we got an empty result
             assert ref_symbols == [] or ref_symbols is None
         except Exception:
@@ -448,15 +454,19 @@ class TestLanguageServerSymbols:
         # Get the containing symbol of a variable in a file
         file_path = os.path.join("test_repo", "services.py")
         # import of typing
-        references_to_typing = language_server.request_referencing_symbols(
-            file_path, 4, 6, include_imports=False, include_file_symbols=True
-        )
+        references_to_typing = [
+            ref.symbol
+            for ref in language_server.request_referencing_symbols(file_path, 4, 6, include_imports=False, include_file_symbols=True)
+        ]
         assert {ref["kind"] for ref in references_to_typing} == {SymbolKind.File}
         assert {ref["body"] for ref in references_to_typing} == {""}
 
         # now include bodies
-        references_to_typing = language_server.request_referencing_symbols(
-            file_path, 4, 6, include_imports=False, include_file_symbols=True, include_body=True
-        )
+        references_to_typing = [
+            ref.symbol
+            for ref in language_server.request_referencing_symbols(
+                file_path, 4, 6, include_imports=False, include_file_symbols=True, include_body=True
+            )
+        ]
         assert {ref["kind"] for ref in references_to_typing} == {SymbolKind.File}
         assert references_to_typing[0]["body"]