Merge pull request #124 from oraios/feature/enhanced-symbol-relations

Enhanced symbol relations and matching

src/multilspy/language_server.py

@@ -38,6 +38,7 @@ from .multilspy_exceptions import MultilspyException
 from .multilspy_logger import MultilspyLogger
 from .multilspy_utils import FileUtils, PathUtils, TextUtils
 from .type_helpers import ensure_all_methods_implemented
+from .lsp_protocol_handler import lsp_types
 
 # Serena dependencies
 # We will need to watch out for circular imports, but it's probably better to not
@@ -582,7 +583,7 @@ class LanguageServer:
         return ret
 
     # Some LS cause problems with this, so the call is isolated from the rest to allow overriding in subclasses
-    async def _send_references_request(self, relative_file_path: str, line: int, column: int):
+    async def _send_references_request(self, relative_file_path: str, line: int, column: int) -> List[lsp_types.Location] | None:
         return await self.server.send.references(
             {
                 "textDocument": {"uri": PathUtils.path_to_uri(os.path.join(self.repository_root_path, relative_file_path))},
@@ -806,8 +807,11 @@ class LanguageServer:
         :param relative_file_path: The relative path of the file that has the symbols
         :param include_body: whether to include the body of the symbols in the result.
         :return: A list of symbols in the file, and a list of root symbols that represent the tree structure of the symbols.
-            Each symbol in hierarchy starting from the roots has a children attribute.
-            All symbols will have a location and a children attribute.
+            All symbols will have a location, a children, and a parent attribute,
+            where the parent attribute is None for root symbols.
+            Note that this is slightly different from the call to request_full_symbol_tree,
+            where the parent attribute will be the file symbol which in turn may have a package symbol as parent.
+            If you need a symbol tree that contains file symbols as well, you should use `request_full_symbol_tree` instead.
         """
         self.logger.log(f"Requesting document symbols for {relative_file_path} for the first time", logging.DEBUG)
         # TODO: it's kinda dumb to not use the cache if include_body is False after include_body was True once
@@ -860,24 +864,32 @@ class LanguageServer:
                     item["selectionRange"] = item["range"]
                 else:
                     item["selectionRange"] = item["location"]["range"]
-            item[LSPConstants.CHILDREN] = item.get(LSPConstants.CHILDREN, [])
+            children = item.get(LSPConstants.CHILDREN, [])
+            for child in children:
+                child["parent"] = item
+            item[LSPConstants.CHILDREN] = children
 
         flat_all_symbol_list: List[multilspy_types.UnifiedSymbolInformation] = []
         assert isinstance(response, list), f"Unexpected response from Language Server: {response}"
         root_nodes: List[multilspy_types.UnifiedSymbolInformation] = []
-        for item in response:
-            if "range" not in item and "location" not in item:
-                if item["kind"] in [SymbolKind.File, SymbolKind.Module]:
+        for root_item in response:
+            if "range" not in root_item and "location" not in root_item:
+                if root_item["kind"] in [SymbolKind.File, SymbolKind.Module]:
                     ...
 
-            turn_item_into_symbol_with_children(item)
-            item = cast(multilspy_types.UnifiedSymbolInformation, item)
-            root_nodes.append(item)
-            assert isinstance(item, dict)
-            assert LSPConstants.NAME in item
-            assert LSPConstants.KIND in item
+            # mutation is more convenient than creating a new dict,
+            # so we cast and rename the var after the mutating call to turn_item_into_symbol_with_children
+            # which turned and item into a "symbol"
+            turn_item_into_symbol_with_children(root_item)
+            root_symbol = cast(multilspy_types.UnifiedSymbolInformation, root_item)
+            root_symbol["parent"] = None
+            
+            root_nodes.append(root_symbol)
+            assert isinstance(root_symbol, dict)
+            assert LSPConstants.NAME in root_symbol
+            assert LSPConstants.KIND in root_symbol
 
-            if LSPConstants.CHILDREN in item:
+            if LSPConstants.CHILDREN in root_symbol:
                 # TODO: l_tree should be a list of TreeRepr. Define the following function to return TreeRepr as well
                 
                 def visit_tree_nodes_and_build_tree_repr(node: GenericDocumentSymbol) -> List[multilspy_types.UnifiedSymbolInformation]:
@@ -891,9 +903,9 @@ class LanguageServer:
                         l.extend(visit_tree_nodes_and_build_tree_repr(child))
                     return l
                 
-                flat_all_symbol_list.extend(visit_tree_nodes_and_build_tree_repr(item))
+                flat_all_symbol_list.extend(visit_tree_nodes_and_build_tree_repr(root_symbol))
             else:
-                flat_all_symbol_list.append(multilspy_types.UnifiedSymbolInformation(**item))
+                flat_all_symbol_list.append(multilspy_types.UnifiedSymbolInformation(**root_symbol))
 
         result = flat_all_symbol_list, root_nodes
         self.logger.log(f"Caching document symbols for {relative_file_path}", logging.DEBUG)
@@ -903,17 +915,19 @@ class LanguageServer:
     
     async def request_full_symbol_tree(self, within_relative_path: str | None = None, include_body: bool = False) -> List[multilspy_types.UnifiedSymbolInformation]:
         """
-        Will go through all files in the project and build a tree of symbols. Note: this may be slow the first time it is called.
+        Will go through all files in the project or within a relative path and build a tree of symbols. 
+        Note: this may be slow the first time it is called, especially if `within_relative_path` is not used to restrict the search.
 
-        For each file, a symbol of kind Module (3) will be created. For directories, a symbol of kind Package (4) will be created.
+        For each file, a symbol of kind File (2) will be created. For directories, a symbol of kind Package (4) will be created.
         All symbols will have a children attribute, thereby representing the tree structure of all symbols in the project
         that are within the repository.
+        All symbols except the root packages will have a parent attribute.
         Will ignore directories starting with '.', language-specific defaults
         and user-configured directories (e.g. from .gitignore).
 
         :param within_relative_path: pass a relative path to only consider symbols within this path.
-                If a file is passed, only the symbols within this file will be considered.
-                If a directory is passed, all files within this directory will be considered.
+            If a file is passed, only the symbols within this file will be considered.
+            If a directory is passed, all files within this directory will be considered.
         :param include_body: whether to include the body of the symbols in the result.
 
         :return: A list of root symbols representing the top-level packages/modules in the project.
@@ -932,17 +946,17 @@ class LanguageServer:
                     return root_nodes
 
         # Helper function to recursively process directories
-        async def process_directory(dir_path: str) -> List[multilspy_types.UnifiedSymbolInformation]:
-            abs_dir_path = self.repository_root_path if dir_path == "." else os.path.join(self.repository_root_path, dir_path)
+        async def process_directory(rel_dir_path: str) -> List[multilspy_types.UnifiedSymbolInformation]:
+            abs_dir_path = self.repository_root_path if rel_dir_path == "." else os.path.join(self.repository_root_path, rel_dir_path)
             abs_dir_path = os.path.realpath(abs_dir_path)
 
             if self.is_ignored_path(str(Path(abs_dir_path).relative_to(self.repository_root_path))):
-                self.logger.log(f"Skipping directory: {dir_path}\n(because it should be ignored)", logging.DEBUG)
+                self.logger.log(f"Skipping directory: {rel_dir_path}\n(because it should be ignored)", logging.DEBUG)
                 return []
 
             result = []
             try:
-                items = os.listdir(abs_dir_path)
+                contained_dir_or_file_names = os.listdir(abs_dir_path)
             except OSError:
                 return []
 
@@ -960,20 +974,45 @@ class LanguageServer:
             )
             result.append(package_symbol)
 
-            for item in items:
-                item_path = os.path.join(abs_dir_path, item)
-                abs_item_path = os.path.join(self.repository_root_path, item_path)
-                rel_item_path = str(Path(abs_item_path).resolve().relative_to(self.repository_root_path))
-                if self.is_ignored_path(rel_item_path):
-                    self.logger.log(f"Skipping item: {rel_item_path}\n(because it should be ignored)", logging.DEBUG)
+            for contained_dir_or_file_name in contained_dir_or_file_names:
+                contained_dir_or_file_abs_path = os.path.join(abs_dir_path, contained_dir_or_file_name)
+                contained_dir_or_file_rel_path = str(Path(contained_dir_or_file_abs_path).resolve().relative_to(self.repository_root_path))
+                if self.is_ignored_path(contained_dir_or_file_rel_path):
+                    self.logger.log(f"Skipping item: {contained_dir_or_file_rel_path}\n(because it should be ignored)", logging.DEBUG)
                     continue
 
-                if os.path.isdir(abs_item_path):
-                    child_symbols = await process_directory(item_path)
+                if os.path.isdir(contained_dir_or_file_abs_path):
+                    child_symbols = await process_directory(contained_dir_or_file_rel_path)
                     package_symbol["children"].extend(child_symbols)
+                    for child in child_symbols:
+                        child["parent"] = package_symbol
+                        
+                elif os.path.isfile(contained_dir_or_file_abs_path):
+                    _, file_root_nodes = await self.request_document_symbols(contained_dir_or_file_rel_path, include_body=include_body)
+                    
+                    # Create file symbol, link with children
+                    file_rel_path = str(Path(contained_dir_or_file_abs_path).resolve().relative_to(self.repository_root_path))
+                    with self.open_file(file_rel_path) as file_data:
+                        fileRange = self._get_range_from_file_content(file_data.contents)
+                    file_symbol = multilspy_types.UnifiedSymbolInformation( # type: ignore
+                        name=os.path.splitext(contained_dir_or_file_name)[0],
+                        kind=multilspy_types.SymbolKind.File,
+                        range=fileRange,
+                        selectionRange=fileRange,
+                        location=multilspy_types.Location(
+                            uri=str(pathlib.Path(contained_dir_or_file_abs_path).as_uri()),
+                            range=fileRange,
+                            absolutePath=str(contained_dir_or_file_abs_path),
+                            relativePath=str(Path(contained_dir_or_file_abs_path).resolve().relative_to(self.repository_root_path)),
+                        ),
+                        children=file_root_nodes,
+                        parent=package_symbol,
+                    )
+                    for child in file_root_nodes:
+                        child["parent"] = file_symbol
 
-                elif os.path.isfile(abs_item_path):
-                    _, root_nodes = await self.request_document_symbols(item_path, include_body=include_body)
+                    # Link file symbol with package
+                    package_symbol["children"].append(file_symbol)
 
                     # TODO: Not sure if this is actually still needed given recent changes to relative path handling
                     def fix_relative_path(nodes: List[multilspy_types.UnifiedSymbolInformation]):
@@ -989,33 +1028,13 @@ class LanguageServer:
                             if "children" in node:
                                 fix_relative_path(node["children"])
 
-                    fix_relative_path(root_nodes)
-
-                    # Create file symbol
-                    file_rel_path = str(Path(abs_item_path).resolve().relative_to(self.repository_root_path))
-                    with self.open_file(file_rel_path) as file_data:
-                        fileRange = self._get_range_from_file_content(file_data.contents)
-                    file_symbol = multilspy_types.UnifiedSymbolInformation( # type: ignore
-                        name=os.path.splitext(item)[0],
-                        kind=multilspy_types.SymbolKind.File,
-                        range=fileRange,
-                        selectionRange=fileRange,
-                        location=multilspy_types.Location(
-                            uri=str(pathlib.Path(abs_item_path).as_uri()),
-                            range=fileRange,
-                            absolutePath=str(abs_item_path),
-                            relativePath=str(Path(abs_item_path).resolve().relative_to(self.repository_root_path)),
-                        ),
-                        children=root_nodes
-                    )
-
-                    package_symbol["children"].append(file_symbol)
+                    fix_relative_path(file_root_nodes)
 
             return result
 
         # Start from the root or the specified directory
-        start_path = within_relative_path or "."
-        return await process_directory(start_path)
+        start_rel_path = within_relative_path or "."
+        return await process_directory(start_rel_path)
 
     @staticmethod
     def _get_range_from_file_content(file_content: str) -> multilspy_types.Range:
@@ -1496,12 +1515,16 @@ class LanguageServer:
 
     async def request_container_of_symbol(self, symbol: multilspy_types.UnifiedSymbolInformation, include_body: bool = False) -> multilspy_types.UnifiedSymbolInformation | None:
         """
-        Finds the container of the given symbol if there is one.
+        Finds the container of the given symbol if there is one. If the parent attribute is present, the parent is returned
+        without further searching.
 
         :param symbol: The symbol to find the container of.
         :param include_body: whether to include the body of the symbol in the result.
+        :return: The container of the given symbol or None if no container is found.
         """
-        assert "location" in symbol
+        if "parent" in symbol:
+            return symbol["parent"]
+        assert "location" in symbol, f"Symbol {symbol} has no location and no parent attribute"
         return await self.request_containing_symbol(
             symbol["location"]["relativePath"],
             symbol["location"]["range"]["start"]["line"],
@@ -1556,7 +1579,7 @@ class LanguageServer:
 
     @property
     def _cache_path(self) -> Path:
-        return Path(self.repository_root_path) / ".serena" / "cache" / "document_symbols_cache.pkl"
+        return Path(self.repository_root_path) / ".serena" / "cache" / "document_symbols_cache_v20-05-25.pkl"
 
     def save_cache(self):
         if self._cache_has_changed:
@@ -1594,7 +1617,7 @@ class LanguageServer:
 
         :param query: The query string to filter symbols by
 
-        :return Union[List[multilspy_types.UnifiedSymbolInformation], None]: A list of matching symbols
+        :return: A list of matching symbols
         """
         response = await self.server.send.workspace_symbol({"query": query})
         if response is None:

src/multilspy/language_servers/gopls/gopls.py

@@ -10,6 +10,7 @@ from typing import AsyncIterator, List
 from overrides import override
 
 from multilspy import multilspy_types
+from multilspy.lsp_protocol_handler import lsp_types
 from multilspy.multilspy_exceptions import MultilspyException
 from multilspy.multilspy_logger import MultilspyLogger
 from multilspy.language_server import LanguageServer
@@ -154,18 +155,3 @@ class Gopls(LanguageServer):
 
             await self.server.shutdown()
             await self.server.stop()
-
-    @override
-    async def _send_references_request(self, relative_file_path: str, line: int, column: int) -> List[multilspy_types.Location]:
-        NUM_COLS_TO_TRY = 10
-        """selectionRange in Gopls always contains the wrong column, 
-        the one at the beginning of the declaration instead of the symbol. We do a dirty hack here (loop over some columns).
-        See https://github.com/golang/go/issues/73521
-        """
-        actual_column = column
-        for actual_column in range(column, column+NUM_COLS_TO_TRY):
-            try:
-                return await super()._send_references_request(relative_file_path, line, actual_column)
-            except (Error, RuntimeError, MultilspyException) as e:
-                self.logger.log(f"Cannot find symbol at {line=}, column={actual_column}, trying to bump column by 1", logging.INFO)
-        raise MultilspyException(f"Failed to find references for symbol in\n{relative_file_path=} at {line=}, {column=}")

src/multilspy/language_servers/gopls/initialize_params.json

@@ -18,6 +18,16 @@
             },
             "definition": {
                 "dynamicRegistration": true
+            },
+            "documentSymbol": {
+                "dynamicRegistration": true,
+                "hierarchicalDocumentSymbolSupport": true,
+                "symbolKind": {
+                    "valueSet": [
+                        1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,
+                        14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26
+                    ]
+                }
             }
         },
         "workspace": {

src/multilspy/multilspy_types.py

@@ -199,7 +199,11 @@ class UnifiedSymbolInformation(TypedDict):
     """ The name of the symbol containing this symbol. This information is for
     user interface purposes (e.g. to render a qualifier in the user interface
     if necessary). It can't be used to re-infer a hierarchy for the document
-    symbols. """
+    symbols. 
+    
+    Note: within Serena, the parent attribute was added and should be used instead. 
+    Most LS don't provide containerName.
+    """
 
     detail: NotRequired[str]
     """ More detail for this symbol, e.g the signature of a function. """
@@ -220,6 +224,11 @@ class UnifiedSymbolInformation(TypedDict):
     Added to be compatible with `lsp_types.DocumentSymbol`, 
     since it is sometimes useful to have the children of the symbol as a user-facing feature."""
     
+    parent: NotRequired[UnifiedSymbolInformation | None]
+    """The parent of the symbol, if there is any. Added with Serena, not part of the LSP.
+    All symbols except the root packages will have a parent.
+    """
+    
 
 class MarkupKind(Enum):
     """Describes the content type that a client supports in various

src/serena/agent.py

@@ -10,11 +10,11 @@ import sys
 import traceback
 from abc import ABC
 from collections import defaultdict
-from collections.abc import Callable, Generator, Iterable
+from collections.abc import Callable, Generator, Iterable, Sequence
 from dataclasses import dataclass, field
 from logging import Logger
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Self, TypeVar, Union, cast
+from typing import TYPE_CHECKING, Any, Self, TypeVar, Union
 
 import yaml
 from sensai.util import logging
@@ -778,7 +778,7 @@ class FindSymbolTool(Tool):
 
     def apply(
         self,
-        name: str,
+        name_path: str,
         depth: int = 0,
         within_relative_path: str | None = None,
         include_body: bool = False,
@@ -788,48 +788,56 @@ class FindSymbolTool(Tool):
         max_answer_chars: int = _DEFAULT_MAX_ANSWER_LENGTH,
     ) -> str:
         """
-        Retrieves information on all symbols/code entities, i.e. classes, methods, attributes, variables, etc.
-        with the given name.
-        The returned symbol location information can subsequently be used to edit the returned symbols
-        or to retrieve further information using other tools.
-        If you already anticipate that you will need to reference children of the symbol (like methods or fields contained in a class),
-        you can specify a depth > 0.
+        Retrieves information on all symbols/code entities (classes, methods, etc.) based on the given `name_path`,
+        which represents a pattern for the symbol's path within the symbol tree of a single file.
+        The returned symbol location can be used for edits or further queries.
+        Specify `depth > 0` to retrieve children (e.g., methods of a class).
 
-        :param name: the name of the symbols to find
-        :param depth: specifies the depth up to which descendants of the symbol are to be retrieved
-            (e.g. depth 1 will retrieve methods and attributes for the case where the symbol refers to a class).
-            Provide a non-zero depth if you intend to subsequently query symbols that are contained in the
-            retrieved symbol.
-        :param within_relative_path: pass a relative path to only consider symbols within this path.
-            If a file is passed, only the symbols within this file will be considered.
-            If a directory is passed, all files within this directory will be considered.
-            If None, the entire codebase will be considered.
-        :param include_body: whether to include the body of all symbols in the result. You should only use this
-            if you actually need the body of the symbol for the task at hand (for example, for a deep analysis
-            of the functionality or for an editing task).
-        :param include_kinds: an optional list of ints representing the LSP symbol kind.
-            If provided, only symbols of the given kinds will be included in the result.
-            Valid kinds:
-            1=file, 2=module, 3=namespace, 4=package, 5=class, 6=method, 7=property, 8=field, 9=constructor, 10=enum,
+        The matching behavior is determined by the structure of `name_path`, which can
+        either be a simple name (e.g. "method") or a name path like "class/method" (relative name path)
+        or "/class/method" (absolute name path). Note that the name path is not a path in the file system
+        but rather a path in the symbol tree **within a single file**. Thus, file or directory names should never
+        be included in the `name_path`. For restricting the search to a single file or directory,
+        the `within_relative_path` parameter should be used instead. The retrieved symbols' `name_path` attribute
+        will always be composed of symbol names, never file or directory names.
+
+        Key aspects of the name path matching behavior:
+        - Trailing slashes in `name_path` play no role and are ignored.
+        - The name of the retrieved symbols will match (either exactly or as a substring)
+          the last segment of `name_path`, while other segments will restrict the search to symbols that
+          have a desired sequence of ancestors.
+        - If there is no starting or intermediate slash in `name_path`, there is no
+          restriction on the ancestor symbols. For example, passing `method` will match
+          against symbols with name paths like `method`, `class/method`, `class/nested_class/method`, etc.
+        - If `name_path` contains a `/` but doesn't start with a `/`, the matching is restricted to symbols
+          with the same ancestors as the last segment of `name_path`. For example, passing `class/method` will match against
+          `class/method` as well as `nested_class/class/method` but not `method`.
+        - If `name_path` starts with a `/`, it will be treated as an absolute name path pattern, meaning
+          that the first segment of it must match the first segment of the symbol's name path.
+          For example, passing `/class` will match only against top-level symbols like `class` but not against `nested_class/class`.
+          Passing `/class/method` will match against `class/method` but not `nested_class/class/method` or `method`.
+
+
+        :param name_path: The name path pattern to search for, see above for details.
+        :param depth: Depth to retrieve descendants (e.g., 1 for class methods/attributes).
+        :param within_relative_path: Optional. Restrict search to this file or directory. If None, searches entire codebase.
+        :param include_body: If True, include the symbol's source code. Use judiciously.
+        :param include_kinds: Optional. List of LSP symbol kind integers to include. (e.g., 5 for Class, 12 for Function).
+            Valid kinds: 1=file, 2=module, 3=namespace, 4=package, 5=class, 6=method, 7=property, 8=field, 9=constructor, 10=enum,
             11=interface, 12=function, 13=variable, 14=constant, 15=string, 16=number, 17=boolean, 18=array, 19=object,
             20=key, 21=null, 22=enum member, 23=struct, 24=event, 25=operator, 26=type parameter
-        :param exclude_kinds: If provided, symbols of the given kinds will be excluded from the result.
-            Takes precedence over include_kinds.
-        :param substring_matching: whether to use substring matching for the symbol name.
-            If True, the symbol name will be matched if it contains the given name as a substring.
-        :param max_answer_chars: if the output is longer than this number of characters,
-            no content will be returned. Don't adjust unless there is really no other way to get the content
-            required for the task. Instead, if the output is too long, you should
-            make a stricter query.
-        :return: a list of symbols (with symbol locations) that match the given name in JSON format
+        :param exclude_kinds: Optional. List of LSP symbol kind integers to exclude. Takes precedence over `include_kinds`.
+        :param substring_matching: If True, use substring matching for the last segment of `name`.
+        :param max_answer_chars: Max characters for the JSON result. If exceeded, no content is returned.
+        :return: JSON string: a list of symbols (with locations) matching the name.
         """
-        include_kinds = cast(list[SymbolKind] | None, include_kinds)
-        exclude_kinds = cast(list[SymbolKind] | None, exclude_kinds)
+        parsed_include_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in include_kinds] if include_kinds else None
+        parsed_exclude_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in exclude_kinds] if exclude_kinds else None
         symbols = self.symbol_manager.find_by_name(
-            name,
+            name_path,
             include_body=include_body,
-            include_kinds=include_kinds,
-            exclude_kinds=exclude_kinds,
+            include_kinds=parsed_include_kinds,
+            exclude_kinds=parsed_exclude_kinds,
             substring_matching=substring_matching,
             within_relative_path=within_relative_path,
         )
@@ -880,13 +888,13 @@ class FindReferencingSymbolsTool(Tool):
             make a stricter query.
         :return: a list of JSON objects with the symbols referencing the requested symbol
         """
-        include_kinds = cast(list[SymbolKind] | None, include_kinds)
-        exclude_kinds = cast(list[SymbolKind] | None, exclude_kinds)
+        parsed_include_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in include_kinds] if include_kinds else None
+        parsed_exclude_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in exclude_kinds] if exclude_kinds else None
         symbols = self.symbol_manager.find_referencing_symbols(
             SymbolLocation(relative_path, line, column),
             include_body=include_body,
-            include_kinds=include_kinds,
-            exclude_kinds=exclude_kinds,
+            include_kinds=parsed_include_kinds,
+            exclude_kinds=parsed_exclude_kinds,
         )
         symbol_dicts = [s.to_dict(kind=True, location=True, depth=0, include_body=include_body) for s in symbols]
         result = json.dumps(symbol_dicts)

src/serena/symbol.py

@@ -1,380 +1,496 @@
-import logging
-import os
-from collections.abc import Iterator, Sequence
-from contextlib import contextmanager
-from copy import copy
-from dataclasses import asdict, dataclass
-from typing import TYPE_CHECKING, Any, Self
-
-from sensai.util.string import ToStringMixin
-
-from multilspy import SyncLanguageServer
-from multilspy.multilspy_types import Position, SymbolKind, UnifiedSymbolInformation
-
-if TYPE_CHECKING:
-    from .agent import SerenaAgent
-
-log = logging.getLogger(__name__)
-
-
-@dataclass
-class SymbolLocation:
-    """
-    Represents the (start) location of a symbol identifier
-    """
-
-    relative_path: str | None
-    """
-    the relative path of the file containing the symbol; if None, the symbol is defined outside of the project's scope
-    """
-    line: int | None
-    """
-    the line number in which the symbol identifier is defined (if the symbol is a function, class, etc.);
-    may be None for some types of symbols (e.g. SymbolKind.File)
-    """
-    column: int | None
-    """
-    the column number in which the symbol identifier is defined (if the symbol is a function, class, etc.);
-    may be None for some types of symbols (e.g. SymbolKind.File)
-    """
-
-    def __post_init__(self) -> None:
-        if self.relative_path is not None:
-            self.relative_path = self.relative_path.replace("/", os.path.sep)
-
-    def to_dict(self) -> dict[str, Any]:
-        return asdict(self)
-
-    def has_position_in_file(self) -> bool:
-        return self.relative_path is not None and self.line is not None and self.column is not None
-
-
-class Symbol(ToStringMixin):
-    def __init__(self, s: UnifiedSymbolInformation) -> None:
-        self.s = s
-
-    def _tostring_includes(self) -> list[str]:
-        return []
-
-    def _tostring_additional_entries(self) -> dict[str, Any]:
-        return dict(name=self.name, kind=self.kind, num_children=len(self.s["children"]))
-
-    @property
-    def name(self) -> str:
-        return self.s["name"]
-
-    @property
-    def kind(self) -> str:
-        return SymbolKind(self.symbol_kind).name
-
-    @property
-    def symbol_kind(self) -> SymbolKind:
-        return self.s["kind"]
-
-    @property
-    def relative_path(self) -> str | None:
-        return self.s["location"]["relativePath"]
-
-    @property
-    def location(self) -> SymbolLocation:
-        """
-        :return: the start location of the actual symbol identifier
-        """
-        return SymbolLocation(relative_path=self.relative_path, line=self.line, column=self.column)
-
-    @property
-    def body_start_position(self) -> Position:
-        return self.s["location"]["range"]["start"]
-
-    @property
-    def body_end_position(self) -> Position:
-        return self.s["location"]["range"]["end"]
-
-    @property
-    def line(self) -> int | None:
-        if "selectionRange" in self.s:
-            return self.s["selectionRange"]["start"]["line"]
-        else:
-            # line is expected to be undefined for some types of symbols (e.g. SymbolKind.File)
-            return None
-
-    @property
-    def column(self) -> int | None:
-        if "selectionRange" in self.s:
-            return self.s["selectionRange"]["start"]["character"]
-        else:
-            # precise location is expected to be undefined for some types of symbols (e.g. SymbolKind.File)
-            return None
-
-    @property
-    def body(self) -> str | None:
-        return self.s.get("body")
-
-    def iter_children(self) -> Iterator["Symbol"]:
-        for c in self.s["children"]:
-            yield Symbol(c)
-
-    def find(
-        self,
-        name: str,
-        substring_matching: bool = False,
-        include_kinds: Sequence[SymbolKind] | None = None,
-        exclude_kinds: Sequence[SymbolKind] | None = None,
-    ) -> list["Symbol"]:
-        result = []
-
-        def should_include(s: "Symbol") -> bool:
-            if not ((substring_matching and name in s.name) or name == s.name):
-                return False
-            if include_kinds is not None and s.symbol_kind not in include_kinds:
-                return False
-            if exclude_kinds is not None and s.symbol_kind in exclude_kinds:
-                return False
-            return True
-
-        def traverse(s: "Symbol") -> None:
-            if should_include(s):
-                result.append(s)
-            for c in s.iter_children():
-                traverse(c)
-
-        traverse(self)
-        return result
-
-    def to_dict(
-        self, kind: bool = False, location: bool = False, depth: int = 0, include_body: bool = False, include_children_body: bool = False
-    ) -> dict[str, Any]:
-        """
-        Convert the symbol to a dictionary.
-
-        :param kind: whether to include the kind of the symbol
-        :param location: whether to include the location of the symbol
-        :param depth: the depth of the symbol
-        :param include_body: whether to include the body of the top-level symbol.
-        :param include_children_body: whether to also include the body of the children.
-            Note that the body of the children is part of the body of the parent symbol,
-            so there is usually no need to set this to True unless you want process the output
-            and pass the children without passing the parent body to the LM.
-        :return: a dictionary representation of the symbol
-        """
-        result: dict[str, Any] = {"name": self.name}
-
-        if kind:
-            result["kind"] = self.kind
-
-        if location:
-            result["location"] = self.location.to_dict()
-
-        if include_body:
-            if self.body is None:
-                log.warning("Requested body for symbol, but it is not present. The symbol might have been loaded with include_body=False.")
-            result["body"] = self.body
-
-        def add_children(s: Self) -> list[dict[str, Any]]:
-            children = []
-            for c in s.iter_children():
-                children.append(
-                    c.to_dict(
-                        kind=kind,
-                        location=location,
-                        depth=depth - 1,
-                        include_body=include_children_body,
-                        include_children_body=include_children_body,
-                    )
-                )
-            return children
-
-        if depth > 0:
-            result["children"] = add_children(self)
-
-        return result
-
-
-class SymbolManager:
-    def __init__(self, lang_server: SyncLanguageServer, agent: "SerenaAgent") -> None:
-        self.lang_server = lang_server
-        self.agent = agent
-
-    def _to_symbols(self, items: list[UnifiedSymbolInformation]) -> list[Symbol]:
-        return [Symbol(s) for s in items]
-
-    def find_by_name(
-        self,
-        name: str,
-        within_relative_path: str | None = None,
-        include_body: bool = False,
-        include_kinds: Sequence[SymbolKind] | None = None,
-        exclude_kinds: Sequence[SymbolKind] | None = None,
-        substring_matching: bool = False,
-    ) -> list[Symbol]:
-        """
-        Find all symbols that match the given name.
-
-        :param name: the name of the symbol to find
-        :param within_relative_path: pass a relative path to only consider symbols within this path.
-            If a file is passed, only the symbols within this file will be considered.
-            If a directory is passed, all files within this directory will be considered.
-            If None, the entire codebase will be considered.
-        :param include_body: whether to include the body of all symbols in the result.
-            Note: you can filter out the bodies of the children if you set include_children_body=False
-            in the to_dict method.
-        :param include_kinds: an optional sequence of ints representing the LSP symbol kind.
-            If provided, only symbols of the given kinds will be included in the result.
-        :param exclude_kinds: If provided, symbols of the given kinds will be excluded from the result.
-            Takes precedence over include_kinds.
-        :param substring_matching: whether to use substring matching for the symbol name.
-            If True, the symbol name will be matched if it contains the given name as a substring.
-        :return: a list of symbols that match the given name
-        """
-        symbols: list[Symbol] = []
-        symbol_roots = self.lang_server.request_full_symbol_tree(within_relative_path=within_relative_path, include_body=include_body)
-        for root in symbol_roots:
-            symbols.extend(
-                Symbol(root).find(name, include_kinds=include_kinds, exclude_kinds=exclude_kinds, substring_matching=substring_matching)
-            )
-        return symbols
-
-    def get_document_symbols(self, relative_path: str) -> list[Symbol]:
-        symbol_dicts, roots = self.lang_server.request_document_symbols(relative_path, include_body=False)
-        symbols = [Symbol(s) for s in symbol_dicts]
-        return symbols
-
-    def find_by_location(self, location: SymbolLocation) -> Symbol | None:
-        if location.relative_path is None:
-            return None
-        symbol_dicts, roots = self.lang_server.request_document_symbols(location.relative_path, include_body=False)
-        for symbol_dict in symbol_dicts:
-            symbol = Symbol(symbol_dict)
-            if symbol.location == location:
-                return symbol
-        return None
-
-    def find_referencing_symbols(
-        self,
-        symbol_location: SymbolLocation,
-        include_body: bool = False,
-        include_kinds: Sequence[SymbolKind] | None = None,
-        exclude_kinds: Sequence[SymbolKind] | None = None,
-    ) -> list[Symbol]:
-        """
-        Find all symbols that reference the symbol at the given location.
-
-        :param symbol_location: the location of the symbol for which to find references
-        :param include_body: whether to include the body of all symbols in the result.
-            Note: you can filter out the bodies of the children if you set include_children_body=False
-            in the to_dict method.
-        :param include_kinds: an optional sequence of ints representing the LSP symbol kind.
-            If provided, only symbols of the given kinds will be included in the result.
-        :param exclude_kinds: If provided, symbols of the given kinds will be excluded from the result.
-            Takes precedence over include_kinds.
-        :return: a list of symbols that reference the given symbol
-        """
-        if not symbol_location.has_position_in_file():
-            raise ValueError("Symbol location does not contain a valid position in a file")
-        assert symbol_location.relative_path is not None
-        assert symbol_location.line is not None
-        assert symbol_location.column is not None
-        symbol_dicts = self.lang_server.request_referencing_symbols(
-            relative_file_path=symbol_location.relative_path,
-            line=symbol_location.line,
-            column=symbol_location.column,
-            include_imports=False,
-            include_self=False,
-            include_body=include_body,
-        )
-
-        if include_kinds is not None:
-            symbol_dicts = [s for s in symbol_dicts if s["kind"] in include_kinds]
-
-        if exclude_kinds is not None:
-            symbol_dicts = [s for s in symbol_dicts if s["kind"] not in exclude_kinds]
-
-        return self._to_symbols(symbol_dicts)
-
-    @contextmanager
-    def _edited_file(self, relative_path: str) -> Iterator[None]:
-        with self.lang_server.open_file(relative_path) as file_buffer:
-            yield
-            root_path = self.lang_server.language_server.repository_root_path
-            abs_path = os.path.join(root_path, relative_path)
-            with open(abs_path, "w", encoding="utf-8") as f:
-                f.write(file_buffer.contents)
-            self.agent.mark_file_modified(relative_path)
-
-    @contextmanager
-    def _edited_symbol_location(self, location: SymbolLocation) -> Iterator[Symbol]:
-        symbol = self.find_by_location(location)
-        if symbol is None:
-            raise ValueError("Symbol not found/has no defined location within a file")
-        assert location.relative_path is not None
-        with self._edited_file(location.relative_path):
-            yield symbol
-
-    def replace_body(self, location: SymbolLocation, body: str) -> None:
-        """
-        Replace the body of the symbol at the given location with the given body
-
-        :param location: the location of the symbol to replace
-        :param body: the new body
-        """
-        # make sure body always ends with at least one newline
-        if not body.endswith("\n"):
-            body += "\n"
-        with self._edited_symbol_location(location) as symbol:
-            assert location.relative_path is not None
-            self.lang_server.delete_text_between_positions(location.relative_path, symbol.body_start_position, symbol.body_end_position)
-            self.lang_server.insert_text_at_position(
-                location.relative_path, symbol.body_start_position["line"], symbol.body_start_position["character"], body
-            )
-
-    def insert_after(self, location: SymbolLocation, body: str) -> None:
-        """
-        Appends content after the given symbol
-
-        :param location: the location of the symbol after which to add new lines
-        :param body: the body of the entity to append
-        """
-        # make sure body always ends with at least one newline
-        if not body.endswith("\n"):
-            body += "\n"
-        with self._edited_symbol_location(location) as symbol:
-            pos = symbol.body_end_position
-            assert location.relative_path is not None
-            self.lang_server.insert_text_at_position(location.relative_path, pos["line"], pos["character"], body)
-
-    def insert_before(self, location: SymbolLocation, body: str) -> None:
-        """
-        Inserts content before the given symbol
-
-        :param location: the location of the symbol before which to add new lines
-        :param body: the body of the entity to insert
-        """
-        # make sure body always ends with at least one newline
-        if not body.endswith("\n"):
-            body += "\n"
-        with self._edited_symbol_location(location) as symbol:
-            pos = copy(symbol.body_start_position)
-            assert location.relative_path is not None
-            self.lang_server.insert_text_at_position(location.relative_path, pos["line"], pos["character"], body)
-
-    def insert_at_line(self, relative_path: str, line: int, content: str) -> None:
-        """
-        Inserts content at the given line in the given file.
-
-        :param line: the 0-based index of the line to insert content at
-        :param content: the content to insert
-        """
-        with self._edited_file(relative_path):
-            self.lang_server.insert_text_at_position(relative_path, line, 0, content)
-
-    def delete_lines(self, relative_path: str, start_line: int, end_line: int) -> None:
-        """
-        Deletes lines in the given file.
-
-        :param start_line: the 0-based index of the first line to delete (inclusive)
-        :param end_line: the 0-based index of the last line to delete (inclusive)
-        """
-        with self._edited_file(relative_path):
-            start_pos = Position(line=start_line, character=0)
-            end_pos = Position(line=end_line + 1, character=0)
-            self.lang_server.delete_text_between_positions(relative_path, start_pos, end_pos)
+import logging
+import os
+from collections.abc import Iterator, Sequence
+from contextlib import contextmanager
+from copy import copy
+from dataclasses import asdict, dataclass
+from typing import TYPE_CHECKING, Any, Self
+
+from sensai.util.string import ToStringMixin
+
+from multilspy import SyncLanguageServer
+from multilspy.multilspy_types import Position, SymbolKind, UnifiedSymbolInformation
+
+if TYPE_CHECKING:
+    from .agent import SerenaAgent
+
+log = logging.getLogger(__name__)
+
+
+@dataclass
+class SymbolLocation:
+    """
+    Represents the (start) location of a symbol identifier
+    """
+
+    relative_path: str | None
+    """
+    the relative path of the file containing the symbol; if None, the symbol is defined outside of the project's scope
+    """
+    line: int | None
+    """
+    the line number in which the symbol identifier is defined (if the symbol is a function, class, etc.);
+    may be None for some types of symbols (e.g. SymbolKind.File)
+    """
+    column: int | None
+    """
+    the column number in which the symbol identifier is defined (if the symbol is a function, class, etc.);
+    may be None for some types of symbols (e.g. SymbolKind.File)
+    """
+
+    def __post_init__(self) -> None:
+        if self.relative_path is not None:
+            self.relative_path = self.relative_path.replace("/", os.path.sep)
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    def has_position_in_file(self) -> bool:
+        return self.relative_path is not None and self.line is not None and self.column is not None
+
+
+class Symbol(ToStringMixin):
+    _NAME_PATH_SEP = "/"
+
+    @staticmethod
+    def match_name_path(
+        name_path: str,
+        symbol_name_path_parts: list[str],
+        substring_matching: bool,
+    ) -> bool:
+        """
+        Checks if a given `name_path` matches a symbol's qualified name parts.
+        See docstring of `Symbol.find` for more details.
+        """
+        assert name_path, "name_path must not be empty"
+        assert symbol_name_path_parts, "symbol_name_path_parts must not be empty"
+        name_path_sep = Symbol._NAME_PATH_SEP
+
+        is_absolute_pattern = name_path.startswith(name_path_sep)
+        pattern_parts = name_path.lstrip(name_path_sep).rstrip(name_path_sep).split(name_path_sep)
+
+        # filtering based on ancestors
+        if len(pattern_parts) > len(symbol_name_path_parts):
+            # can't possibly match if pattern has more parts than symbol
+            return False
+        if is_absolute_pattern and len(pattern_parts) != len(symbol_name_path_parts):
+            # for absolute patterns, the number of parts must match exactly
+            return False
+        if symbol_name_path_parts[-len(pattern_parts) : -1] != pattern_parts[:-1]:
+            # ancestors must match
+            return False
+
+        # matching the last part of the symbol name
+        name_to_match = pattern_parts[-1]
+        symbol_name = symbol_name_path_parts[-1]
+        if substring_matching:
+            return name_to_match in symbol_name
+        else:
+            return name_to_match == symbol_name
+
+    def __init__(self, symbol_root_from_ls: UnifiedSymbolInformation) -> None:
+        self.symbol_root = symbol_root_from_ls
+
+    def _tostring_includes(self) -> list[str]:
+        return []
+
+    def _tostring_additional_entries(self) -> dict[str, Any]:
+        return dict(name=self.name, kind=self.kind, num_children=len(self.symbol_root["children"]))
+
+    @property
+    def name(self) -> str:
+        return self.symbol_root["name"]
+
+    @property
+    def kind(self) -> str:
+        return SymbolKind(self.symbol_kind).name
+
+    @property
+    def symbol_kind(self) -> SymbolKind:
+        return self.symbol_root["kind"]
+
+    @property
+    def relative_path(self) -> str | None:
+        location = self.symbol_root.get("location")
+        if location:
+            return location.get("relativePath")
+        return None
+
+    @property
+    def location(self) -> SymbolLocation:
+        """
+        :return: the start location of the actual symbol identifier
+        """
+        return SymbolLocation(relative_path=self.relative_path, line=self.line, column=self.column)
+
+    @property
+    def body_start_position(self) -> Position | None:
+        location = self.symbol_root.get("location")
+        if location:
+            range_info = location.get("range")
+            if range_info:
+                start_pos = range_info.get("start")
+                if start_pos:
+                    return start_pos
+        return None
+
+    @property
+    def body_end_position(self) -> Position | None:
+        location = self.symbol_root.get("location")
+        if location:
+            range_info = location.get("range")
+            if range_info:
+                end_pos = range_info.get("end")
+                if end_pos:
+                    return end_pos
+        return None
+
+    @property
+    def line(self) -> int | None:
+        if "selectionRange" in self.symbol_root:
+            return self.symbol_root["selectionRange"]["start"]["line"]
+        else:
+            # line is expected to be undefined for some types of symbols (e.g. SymbolKind.File)
+            return None
+
+    @property
+    def column(self) -> int | None:
+        if "selectionRange" in self.symbol_root:
+            return self.symbol_root["selectionRange"]["start"]["character"]
+        else:
+            # precise location is expected to be undefined for some types of symbols (e.g. SymbolKind.File)
+            return None
+
+    @property
+    def body(self) -> str | None:
+        return self.symbol_root.get("body")
+
+    def get_name_path(self) -> str:
+        """
+        Get the name path of the symbol (e.g. "class/method/inner_function").
+        """
+        return self._NAME_PATH_SEP.join(self.get_name_path_parts())
+
+    def get_name_path_parts(self) -> list[str]:
+        """
+        Get the parts of the name path of the symbol (e.g. ["class", "method", "inner_function"]).
+        """
+        ancestors_within_file = list(self.iter_ancestors(up_to_symbol_kind=SymbolKind.File))
+        ancestors_within_file.reverse()
+        return [a.name for a in ancestors_within_file] + [self.name]
+
+    def iter_children(self) -> Iterator[Self]:
+        for c in self.symbol_root["children"]:
+            yield self.__class__(c)
+
+    def iter_ancestors(self, up_to_symbol_kind: SymbolKind | None = None) -> Iterator[Self]:
+        """
+        Iterate over all ancestors of the symbol, starting with the parent and going up to the root or
+        the given symbol kind.
+
+        :param up_to_symbol_kind: if provided, iteration will stop *before* the first ancestor of the given kind.
+            A typical use case is to pass `SymbolKind.File` or `SymbolKind.Package`.
+        """
+        parent = self.get_parent()
+        if parent is not None:
+            if up_to_symbol_kind is None or parent.symbol_kind != up_to_symbol_kind:
+                yield parent
+                yield from parent.iter_ancestors(up_to_symbol_kind=up_to_symbol_kind)
+
+    def get_parent(self) -> Self | None:
+        parent_root = self.symbol_root.get("parent")
+        if parent_root is None:
+            return None
+        return self.__class__(parent_root)
+
+    def find(
+        self,
+        name_path: str,
+        substring_matching: bool = False,
+        include_kinds: Sequence[SymbolKind] | None = None,
+        exclude_kinds: Sequence[SymbolKind] | None = None,
+    ) -> list[Self]:
+        """
+        Find all symbols within the symbol's subtree that match the given `name_path`.
+        The matching behavior is determined by the structure of `name_path`, which can
+        either be a simple name (e.g. "method") or a name path like "class/method" (relative name path)
+        or "/class/method" (absolute name path).
+
+        Key aspects of the name path matching behavior:
+        - Trailing slashes in `name_path` play no role and are ignored.
+        - The name of the retrieved symbols will match (either exactly or as a substring)
+          the last segment of `name_path`, while other segments will restrict the search to symbols that
+          have a desired sequence of ancestors.
+        - If there is no starting or intermediate slash in `name_path`, there is no
+          restriction on the ancestor symbols. For example, passing `method` will match
+          against symbols with name paths like `method`, `class/method`, `class/nested_class/method`, etc.
+        - If `name_path` contains a `/` but doesn't start with a `/`, the matching is restricted to symbols
+          with the same ancestors as the last segment of `name_path`. For example, passing `class/method` will match against
+          `class/method` as well as `nested_class/class/method` but not `method`.
+        - If `name_path` starts with a `/`, it will be treated as an absolute name path pattern, meaning
+          that the first segment of it must match the first segment of the symbol's name path.
+          For example, passing `/class` will match only against top-level symbols like `class` but not against `nested_class/class`.
+          Passing `/class/method` will match against `class/method` but not `nested_class/class/method` or `method`.
+
+        :param name_path: the name path to match against
+        :param substring_matching: whether to use substring matching (as opposed to exact matching)
+            of the last segment of `name_path` against the symbol name.
+        :param include_kinds: an optional sequence of ints representing the LSP symbol kind.
+            If provided, only symbols of the given kinds will be included in the result.
+        :param exclude_kinds: If provided, symbols of the given kinds will be excluded from the result.
+        """
+        result = []
+
+        def should_include(s: "Symbol") -> bool:
+            if include_kinds is not None and s.symbol_kind not in include_kinds:
+                return False
+            if exclude_kinds is not None and s.symbol_kind in exclude_kinds:
+                return False
+            return Symbol.match_name_path(
+                name_path=name_path,
+                symbol_name_path_parts=s.get_name_path_parts(),
+                substring_matching=substring_matching,
+            )
+
+        def traverse(s: "Symbol") -> None:
+            if should_include(s):
+                result.append(s)
+            for c in s.iter_children():
+                traverse(c)
+
+        traverse(self)
+        return result
+
+    def to_dict(
+        self, kind: bool = False, location: bool = False, depth: int = 0, include_body: bool = False, include_children_body: bool = False
+    ) -> dict[str, Any]:
+        """
+        Convert the symbol to a dictionary.
+
+        :param kind: whether to include the kind of the symbol
+        :param location: whether to include the location of the symbol
+        :param depth: the depth of the symbol
+        :param include_body: whether to include the body of the top-level symbol.
+        :param include_children_body: whether to also include the body of the children.
+            Note that the body of the children is part of the body of the parent symbol,
+            so there is usually no need to set this to True unless you want process the output
+            and pass the children without passing the parent body to the LM.
+        :return: a dictionary representation of the symbol
+        """
+        result: dict[str, Any] = {"name": self.name, "name_path": self.get_name_path()}
+
+        if kind:
+            result["kind"] = self.kind
+
+        if location:
+            result["location"] = self.location.to_dict()
+
+        if include_body:
+            if self.body is None:
+                log.warning("Requested body for symbol, but it is not present. The symbol might have been loaded with include_body=False.")
+            result["body"] = self.body
+
+        def add_children(s: Self) -> list[dict[str, Any]]:
+            children = []
+            for c in s.iter_children():
+                children.append(
+                    c.to_dict(
+                        kind=kind,
+                        location=location,
+                        depth=depth - 1,
+                        include_body=include_children_body,
+                        include_children_body=include_children_body,
+                    )
+                )
+            return children
+
+        if depth > 0:
+            result["children"] = add_children(self)
+
+        return result
+
+
+class SymbolManager:
+    def __init__(self, lang_server: SyncLanguageServer, agent: "SerenaAgent") -> None:
+        self.lang_server = lang_server
+        self.agent = agent
+
+    def _to_symbols(self, items: list[UnifiedSymbolInformation]) -> list[Symbol]:
+        return [Symbol(s) for s in items]
+
+    def find_by_name(
+        self,
+        name: str,
+        include_body: bool = False,
+        include_kinds: Sequence[SymbolKind] | None = None,
+        exclude_kinds: Sequence[SymbolKind] | None = None,
+        substring_matching: bool = False,
+        within_relative_path: str | None = None,
+    ) -> list[Symbol]:
+        """
+        Find all symbols that match the given name. See docstring of `Symbol.find` for more details.
+        The only parameter not mentioned there is `within_relative_path`, which can be used to restrict the search
+        to symbols within a specific file or directory.
+        """
+        symbols: list[Symbol] = []
+        symbol_roots = self.lang_server.request_full_symbol_tree(within_relative_path=within_relative_path, include_body=include_body)
+        for root in symbol_roots:
+            symbols.extend(
+                Symbol(root).find(name, include_kinds=include_kinds, exclude_kinds=exclude_kinds, substring_matching=substring_matching)
+            )
+        return symbols
+
+    def get_document_symbols(self, relative_path: str) -> list[Symbol]:
+        symbol_dicts, roots = self.lang_server.request_document_symbols(relative_path, include_body=False)
+        symbols = [Symbol(s) for s in symbol_dicts]
+        return symbols
+
+    def find_by_location(self, location: SymbolLocation) -> Symbol | None:
+        if location.relative_path is None:
+            return None
+        symbol_dicts, roots = self.lang_server.request_document_symbols(location.relative_path, include_body=False)
+        for symbol_dict in symbol_dicts:
+            symbol = Symbol(symbol_dict)
+            if symbol.location == location:
+                return symbol
+        return None
+
+    def find_referencing_symbols(
+        self,
+        symbol_location: SymbolLocation,
+        include_body: bool = False,
+        include_kinds: Sequence[SymbolKind] | None = None,
+        exclude_kinds: Sequence[SymbolKind] | None = None,
+    ) -> list[Symbol]:
+        """
+        Find all symbols that reference the symbol at the given location.
+
+        :param symbol_location: the location of the symbol for which to find references
+        :param include_body: whether to include the body of all symbols in the result.
+            Note: you can filter out the bodies of the children if you set include_children_body=False
+            in the to_dict method.
+        :param include_kinds: an optional sequence of ints representing the LSP symbol kind.
+            If provided, only symbols of the given kinds will be included in the result.
+        :param exclude_kinds: If provided, symbols of the given kinds will be excluded from the result.
+            Takes precedence over include_kinds.
+        :return: a list of symbols that reference the given symbol
+        """
+        if not symbol_location.has_position_in_file():
+            raise ValueError("Symbol location does not contain a valid position in a file")
+        assert symbol_location.relative_path is not None
+        assert symbol_location.line is not None
+        assert symbol_location.column is not None
+        symbol_dicts = self.lang_server.request_referencing_symbols(
+            relative_file_path=symbol_location.relative_path,
+            line=symbol_location.line,
+            column=symbol_location.column,
+            include_imports=False,
+            include_self=False,
+            include_body=include_body,
+        )
+
+        if include_kinds is not None:
+            symbol_dicts = [s for s in symbol_dicts if s["kind"] in include_kinds]
+
+        if exclude_kinds is not None:
+            symbol_dicts = [s for s in symbol_dicts if s["kind"] not in exclude_kinds]
+
+        return self._to_symbols(symbol_dicts)
+
+    @contextmanager
+    def _edited_file(self, relative_path: str) -> Iterator[None]:
+        with self.lang_server.open_file(relative_path) as file_buffer:
+            yield
+            root_path = self.lang_server.language_server.repository_root_path
+            abs_path = os.path.join(root_path, relative_path)
+            with open(abs_path, "w", encoding="utf-8") as f:
+                f.write(file_buffer.contents)
+            self.agent.mark_file_modified(relative_path)
+
+    @contextmanager
+    def _edited_symbol_location(self, location: SymbolLocation) -> Iterator[Symbol]:
+        symbol = self.find_by_location(location)
+        if symbol is None:
+            raise ValueError("Symbol not found/has no defined location within a file")
+        assert location.relative_path is not None
+        with self._edited_file(location.relative_path):
+            yield symbol
+
+    def replace_body(self, location: SymbolLocation, body: str) -> None:
+        """
+        Replace the body of the symbol at the given location with the given body
+
+        :param location: the location of the symbol to replace
+        :param body: the new body
+        """
+        # make sure body always ends with at least one newline
+        if not body.endswith("\n"):
+            body += "\n"
+        with self._edited_symbol_location(location) as symbol:
+            assert location.relative_path is not None
+            start_pos = symbol.body_start_position
+            end_pos = symbol.body_end_position
+            if start_pos is None or end_pos is None:
+                raise ValueError(f"Symbol at {location} does not have a defined body range.")
+            # At this point, start_pos and end_pos are guaranteed to be Position objects
+            self.lang_server.delete_text_between_positions(location.relative_path, start_pos, end_pos)
+            self.lang_server.insert_text_at_position(location.relative_path, start_pos["line"], start_pos["character"], body)
+
+    def insert_after(self, location: SymbolLocation, body: str) -> None:
+        """
+        Appends content after the given symbol
+
+        :param location: the location of the symbol after which to add new lines
+        :param body: the body of the entity to append
+        """
+        # make sure body always ends with at least one newline
+        if not body.endswith("\n"):
+            body += "\n"
+        with self._edited_symbol_location(location) as symbol:
+            pos = symbol.body_end_position
+            if pos is None:
+                raise ValueError(f"Symbol at {location} does not have a defined end position.")
+            # At this point, pos is guaranteed to be a Position object
+            assert location.relative_path is not None
+            self.lang_server.insert_text_at_position(location.relative_path, pos["line"], pos["character"], body)
+
+    def insert_before(self, location: SymbolLocation, body: str) -> None:
+        """
+        Inserts content before the given symbol
+
+        :param location: the location of the symbol before which to add new lines
+        :param body: the body of the entity to insert
+        """
+        # make sure body always ends with at least one newline
+        if not body.endswith("\n"):
+            body += "\n"
+        with self._edited_symbol_location(location) as symbol:
+            original_start_pos = symbol.body_start_position
+            if original_start_pos is None:
+                raise ValueError(f"Symbol at {location} does not have a defined start position.")
+            # At this point, original_start_pos is guaranteed to be a Position object, so copying is safe.
+            pos = copy(original_start_pos)
+            assert location.relative_path is not None
+            self.lang_server.insert_text_at_position(location.relative_path, pos["line"], pos["character"], body)
+
+    def insert_at_line(self, relative_path: str, line: int, content: str) -> None:
+        """
+        Inserts content at the given line in the given file.
+
+        :param line: the 0-based index of the line to insert content at
+        :param content: the content to insert
+        """
+        with self._edited_file(relative_path):
+            self.lang_server.insert_text_at_position(relative_path, line, 0, content)
+
+    def delete_lines(self, relative_path: str, start_line: int, end_line: int) -> None:
+        """
+        Deletes lines in the given file.
+
+        :param start_line: the 0-based index of the first line to delete (inclusive)
+        :param end_line: the 0-based index of the last line to delete (inclusive)
+        """
+        with self._edited_file(relative_path):
+            start_pos = Position(line=start_line, character=0)
+            end_pos = Position(line=end_line + 1, character=0)
+            self.lang_server.delete_text_between_positions(relative_path, start_pos, end_pos)

test/__init__.py

@@ -0,0 +1 @@
+

test/multilspy/go/test_go_basic.py

@@ -7,6 +7,7 @@ from multilspy.multilspy_config import Language
 from multilspy.multilspy_utils import SymbolUtils
 
 
+@pytest.mark.go
 class TestGoLanguageServer:
     @pytest.mark.parametrize("language_server", [Language.GO], indirect=True)
     def test_find_symbol(self, language_server: SyncLanguageServer) -> None:

test/serena/__init__.py

@@ -0,0 +1 @@
+

test/serena/test_serena_agent.py

@@ -87,3 +87,132 @@ class TestSerenaAgent:
         assert any(
             ref["location"]["relative_path"] == ref_file for ref in refs
         ), f"Expected to find reference to {symbol_name} in {ref_file} for {agent.project_config.language.name}. refs={refs}"
+
+    @pytest.mark.parametrize(
+        "serena_agent,name_path,substring_matching,expected_symbol_name,expected_kind,expected_file",
+        [
+            pytest.param(
+                Language.PYTHON,
+                "OuterClass/NestedClass",
+                False,
+                "NestedClass",
+                "Class",
+                os.path.join("test_repo", "nested.py"),
+                id="exact_qualname_class",
+                marks=pytest.mark.python,
+            ),
+            pytest.param(
+                Language.PYTHON,
+                "OuterClass/NestedClass/find_me",
+                False,
+                "find_me",
+                "Method",
+                os.path.join("test_repo", "nested.py"),
+                id="exact_qualname_method",
+                marks=pytest.mark.python,
+            ),
+            pytest.param(
+                Language.PYTHON,
+                "OuterClass/NestedCl",  # Substring for NestedClass
+                True,
+                "NestedClass",
+                "Class",
+                os.path.join("test_repo", "nested.py"),
+                id="substring_qualname_class",
+                marks=pytest.mark.python,
+            ),
+            pytest.param(
+                Language.PYTHON,
+                "OuterClass/NestedClass/find_m",  # Substring for find_me
+                True,
+                "find_me",
+                "Method",
+                os.path.join("test_repo", "nested.py"),
+                id="substring_qualname_method",
+                marks=pytest.mark.python,
+            ),
+            pytest.param(
+                Language.PYTHON,
+                "/OuterClass",  # Absolute path
+                False,
+                "OuterClass",
+                "Class",
+                os.path.join("test_repo", "nested.py"),
+                id="absolute_qualname_class",
+                marks=pytest.mark.python,
+            ),
+            pytest.param(
+                Language.PYTHON,
+                "/OuterClass/NestedClass/find_m",  # Absolute path with substring
+                True,
+                "find_me",
+                "Method",
+                os.path.join("test_repo", "nested.py"),
+                id="absolute_substring_qualname_method",
+                marks=pytest.mark.python,
+            ),
+        ],
+        indirect=["serena_agent"],
+    )
+    def test_find_symbol_name_path(
+        self,
+        serena_agent: SerenaAgent,
+        name_path: str,
+        substring_matching: bool,
+        expected_symbol_name: str,
+        expected_kind: str,
+        expected_file: str,
+    ):
+        agent = serena_agent
+        find_symbol_tool = agent.get_tool(FindSymbolTool)
+        result = find_symbol_tool.apply(
+            name_path=name_path,
+            depth=0,
+            within_relative_path=None,
+            include_body=False,
+            include_kinds=None,
+            exclude_kinds=None,
+            substring_matching=substring_matching,
+        )
+        symbols = json.loads(result)
+        assert any(
+            expected_symbol_name == s["name"]
+            and expected_kind.lower() in s["kind"].lower()
+            and expected_file in s["location"]["relative_path"]
+            for s in symbols
+        ), f"Expected to find {name_path} ({expected_kind}) in {expected_file} for {agent.project_config.language.name}. Symbols: {symbols}"
+
+    @pytest.mark.parametrize(
+        "serena_agent,name_path",
+        [
+            pytest.param(
+                Language.PYTHON,
+                "/NestedClass",  # Absolute path, NestedClass is not top-level
+                id="absolute_path_non_top_level_no_match",
+                marks=pytest.mark.python,
+            ),
+            pytest.param(
+                Language.PYTHON,
+                "/NoSuchParent/NestedClass",  # Absolute path with non-existent parent
+                id="absolute_path_non_existent_parent_no_match",
+                marks=pytest.mark.python,
+            ),
+        ],
+        indirect=["serena_agent"],
+    )
+    def test_find_symbol_name_path_no_match(
+        self,
+        serena_agent: SerenaAgent,
+        name_path: str,
+    ):
+        agent = serena_agent
+        find_symbol_tool = agent.get_tool(FindSymbolTool)
+        result = find_symbol_tool.apply(
+            name_path=name_path,
+            depth=0,
+            substring_matching=True,
+        )
+        symbols = json.loads(result)
+        assert (
+            not symbols
+        ), f"Expected to find no symbols for {name_path} for {agent.project_config.language.name}. Symbols found: {symbols}"

test/serena/test_symbol.py

@@ -0,0 +1,162 @@
+import pytest
+
+from src.serena.symbol import Symbol
+
+
+class TestSymbolNameMatching:
+    def _create_assertion_error_message(
+        self,
+        name_path_pattern: str,
+        symbol_name_path_parts: list[str],
+        is_substring_match: bool,
+        expected_result: bool,
+        actual_result: bool,
+    ) -> str:
+        """Helper to create a detailed error message for assertions."""
+        qnp_repr = "/".join(symbol_name_path_parts)
+
+        return (
+            f"Pattern '{name_path_pattern}' (substring: {is_substring_match}) vs "
+            f"Qualname parts {symbol_name_path_parts} (as '{qnp_repr}'). "
+            f"Expected: {expected_result}, Got: {actual_result}"
+        )
+
+    @pytest.mark.parametrize(
+        "name_path_pattern, symbol_name_path_parts, is_substring_match, expected",
+        [
+            # Exact matches, anywhere in the name (is_substring_match=False)
+            pytest.param("foo", ["foo"], False, True, id="'foo' matches 'foo' exactly (simple)"),
+            pytest.param("foo/", ["foo"], False, True, id="'foo/' matches 'foo' exactly (simple)"),
+            pytest.param("foo", ["bar", "foo"], False, True, id="'foo' matches ['bar', 'foo'] exactly (simple, last element)"),
+            pytest.param("foo", ["foobar"], False, False, id="'foo' does not match 'foobar' exactly (simple)"),
+            pytest.param(
+                "foo", ["bar", "foobar"], False, False, id="'foo' does not match ['bar', 'foobar'] exactly (simple, last element)"
+            ),
+            pytest.param(
+                "foo", ["path", "to", "foo"], False, True, id="'foo' matches ['path', 'to', 'foo'] exactly (simple, last element)"
+            ),
+            # Exact matches, absolute patterns (is_substring_match=False)
+            pytest.param("/foo", ["foo"], False, True, id="'/foo' matches ['foo'] exactly (absolute simple)"),
+            pytest.param("/foo", ["foo", "bar"], False, False, id="'/foo' does not match ['foo', 'bar'] (absolute simple, len mismatch)"),
+            pytest.param("/foo", ["bar"], False, False, id="'/foo' does not match ['bar'] (absolute simple, name mismatch)"),
+            pytest.param(
+                "/foo", ["bar", "foo"], False, False, id="'/foo' does not match ['bar', 'foo'] (absolute simple, position mismatch)"
+            ),
+            # Substring matches, anywhere in the name (is_substring_match=True)
+            pytest.param("foo", ["foobar"], True, True, id="'foo' matches 'foobar' as substring (simple)"),
+            pytest.param("foo", ["bar", "foobar"], True, True, id="'foo' matches ['bar', 'foobar'] as substring (simple, last element)"),
+            pytest.param(
+                "foo", ["barfoo"], True, True, id="'foo' matches 'barfoo' as substring (simple)"
+            ),  # This was potentially ambiguous before
+            pytest.param("foo", ["baz"], True, False, id="'foo' does not match 'baz' as substring (simple)"),
+            pytest.param("foo", ["bar", "baz"], True, False, id="'foo' does not match ['bar', 'baz'] as substring (simple, last element)"),
+            pytest.param("foo", ["my_foobar_func"], True, True, id="'foo' matches 'my_foobar_func' as substring (simple)"),
+            pytest.param(
+                "foo",
+                ["ClassA", "my_foobar_method"],
+                True,
+                True,
+                id="'foo' matches ['ClassA', 'my_foobar_method'] as substring (simple, last element)",
+            ),
+            pytest.param("foo", ["my_bar_func"], True, False, id="'foo' does not match 'my_bar_func' as substring (simple)"),
+            # Substring matches, absolute patterns (is_substring_match=True)
+            pytest.param("/foo", ["foobar"], True, True, id="'/foo' matches ['foobar'] as substring (absolute simple)"),
+            pytest.param("/foo/", ["foobar"], True, True, id="'/foo/' matches ['foobar'] as substring (absolute simple, last element)"),
+            pytest.param("/foo", ["barfoobaz"], True, True, id="'/foo' matches ['barfoobaz'] as substring (absolute simple)"),
+            pytest.param(
+                "/foo", ["foo", "bar"], True, False, id="'/foo' does not match ['foo', 'bar'] as substring (absolute simple, len mismatch)"
+            ),
+            pytest.param("/foo", ["bar"], True, False, id="'/foo' does not match ['bar'] (absolute simple, no substr)"),
+            pytest.param(
+                "/foo", ["bar", "foo"], True, False, id="'/foo' does not match ['bar', 'foo'] (absolute simple, position mismatch)"
+            ),
+            pytest.param(
+                "/foo/", ["bar", "foo"], True, False, id="'/foo/' does not match ['bar', 'foo'] (absolute simple, position mismatch)"
+            ),
+        ],
+    )
+    def test_match_simple_name(self, name_path_pattern, symbol_name_path_parts, is_substring_match, expected):
+        """Tests matching for simple names (no '/' in pattern)."""
+        result = Symbol.match_name_path(name_path_pattern, symbol_name_path_parts, is_substring_match)
+        error_msg = self._create_assertion_error_message(name_path_pattern, symbol_name_path_parts, is_substring_match, expected, result)
+        assert result == expected, error_msg
+
+    @pytest.mark.parametrize(
+        "name_path_pattern, symbol_name_path_parts, is_substring_match, expected",
+        [
+            # --- Relative patterns (suffix matching) ---
+            # Exact matches, relative patterns (is_substring_match=False)
+            pytest.param("bar/foo", ["bar", "foo"], False, True, id="R: 'bar/foo' matches ['bar', 'foo'] exactly"),
+            pytest.param("bar/foo", ["mod", "bar", "foo"], False, True, id="R: 'bar/foo' matches ['mod', 'bar', 'foo'] exactly (suffix)"),
+            pytest.param(
+                "bar/foo", ["bar", "foo", "baz"], False, False, id="R: 'bar/foo' does not match ['bar', 'foo', 'baz'] (pattern shorter)"
+            ),
+            pytest.param("bar/foo", ["bar"], False, False, id="R: 'bar/foo' does not match ['bar'] (pattern longer)"),
+            pytest.param("bar/foo", ["baz", "foo"], False, False, id="R: 'bar/foo' does not match ['baz', 'foo'] (first part mismatch)"),
+            pytest.param("bar/foo", ["bar", "baz"], False, False, id="R: 'bar/foo' does not match ['bar', 'baz'] (last part mismatch)"),
+            pytest.param("bar/foo", ["foo"], False, False, id="R: 'bar/foo' does not match ['foo'] (pattern longer)"),
+            pytest.param(
+                "bar/foo", ["other", "foo"], False, False, id="R: 'bar/foo' does not match ['other', 'foo'] (first part mismatch)"
+            ),
+            pytest.param(
+                "bar/foo", ["bar", "otherfoo"], False, False, id="R: 'bar/foo' does not match ['bar', 'otherfoo'] (last part mismatch)"
+            ),
+            # Substring matches, relative patterns (is_substring_match=True)
+            pytest.param("bar/foo", ["bar", "foobar"], True, True, id="R: 'bar/foo' matches ['bar', 'foobar'] as substring"),
+            pytest.param(
+                "bar/foo", ["mod", "bar", "foobar"], True, True, id="R: 'bar/foo' matches ['mod', 'bar', 'foobar'] as substring (suffix)"
+            ),
+            pytest.param("bar/foo", ["bar", "bazfoo"], True, True, id="R: 'bar/foo' matches ['bar', 'bazfoo'] as substring"),
+            pytest.param("bar/fo", ["bar", "foo"], True, True, id="R: 'bar/fo' matches ['bar', 'foo'] as substring"),
+            pytest.param("bar/foo", ["bar", "baz"], True, False, id="R: 'bar/foo' does not match ['bar', 'baz'] (last no substr)"),
+            pytest.param(
+                "bar/foo", ["baz", "foobar"], True, False, id="R: 'bar/foo' does not match ['baz', 'foobar'] (first part mismatch)"
+            ),
+            pytest.param(
+                "bar/foo", ["bar", "my_foobar_method"], True, True, id="R: 'bar/foo' matches ['bar', 'my_foobar_method'] as substring"
+            ),
+            pytest.param(
+                "bar/foo",
+                ["mod", "bar", "my_foobar_method"],
+                True,
+                True,
+                id="R: 'bar/foo' matches ['mod', 'bar', 'my_foobar_method'] as substring (suffix)",
+            ),
+            pytest.param(
+                "bar/foo",
+                ["bar", "another_method"],
+                True,
+                False,
+                id="R: 'bar/foo' does not match ['bar', 'another_method'] (last no substr)",
+            ),
+            pytest.param(
+                "bar/foo",
+                ["other", "my_foobar_method"],
+                True,
+                False,
+                id="R: 'bar/foo' does not match ['other', 'my_foobar_method'] (first part mismatch)",
+            ),
+            pytest.param("bar/f", ["bar", "foo"], True, True, id="R: 'bar/f' matches ['bar', 'foo'] as substring"),
+            # Exact matches, absolute patterns (is_substring_match=False)
+            pytest.param("/bar/foo", ["bar", "foo"], False, True, id="A: '/bar/foo' matches ['bar', 'foo'] exactly"),
+            pytest.param(
+                "/bar/foo", ["bar", "foo", "baz"], False, False, id="A: '/bar/foo' does not match ['bar', 'foo', 'baz'] (pattern shorter)"
+            ),
+            pytest.param("/bar/foo", ["bar"], False, False, id="A: '/bar/foo' does not match ['bar'] (pattern longer)"),
+            pytest.param("/bar/foo", ["baz", "foo"], False, False, id="A: '/bar/foo' does not match ['baz', 'foo'] (first part mismatch)"),
+            pytest.param("/bar/foo", ["bar", "baz"], False, False, id="A: '/bar/foo' does not match ['bar', 'baz'] (last part mismatch)"),
+            # Substring matches (is_substring_match=True)
+            pytest.param("/bar/foo", ["bar", "foobar"], True, True, id="A: '/bar/foo' matches ['bar', 'foobar'] as substring"),
+            pytest.param("/bar/foo", ["bar", "bazfoo"], True, True, id="A: '/bar/foo' matches ['bar', 'bazfoo'] as substring"),
+            pytest.param("/bar/fo", ["bar", "foo"], True, True, id="A: '/bar/fo' matches ['bar', 'foo'] as substring"),
+            pytest.param("/bar/foo", ["bar", "baz"], True, False, id="A: '/bar/foo' does not match ['bar', 'baz'] (last no substr)"),
+            pytest.param(
+                "/bar/foo", ["baz", "foobar"], True, False, id="A: '/bar/foo' does not match ['baz', 'foobar'] (first part mismatch)"
+            ),
+        ],
+    )
+    def test_match_name_path_pattern_path_len_2(self, name_path_pattern, symbol_name_path_parts, is_substring_match, expected):
+        """Tests matching for qualified names (e.g. 'module/class/func')."""
+        result = Symbol.match_name_path(name_path_pattern, symbol_name_path_parts, is_substring_match)
+        error_msg = self._create_assertion_error_message(name_path_pattern, symbol_name_path_parts, is_substring_match, expected, result)
+        assert result == expected, error_msg