Merge branch 'main' of github.com:oraios/serena

Conflicts:
	src/serena/mcp.py

.vscode/settings.json

@@ -1,5 +1,6 @@
 {
     "cSpell.words": [
+        "langsrv",
         "multilspy",
         "pixi"
     ]

src/multilspy/language_server.py

@@ -6,6 +6,7 @@ The details of Language Specific configuration are not exposed to the user.
 """
 
 import asyncio
+from collections import defaultdict
 from copy import copy
 import dataclasses
 from fnmatch import fnmatch
@@ -742,7 +743,7 @@ class LanguageServer:
         self._cache_has_changed = True
         return result
     
-    async def request_full_symbol_tree(self, start_package_relative_path: str | None = None, include_body: bool = False) -> List[multilspy_types.UnifiedSymbolInformation]:
+    async def request_full_symbol_tree(self, start_dir_relative_path: str | None = None, include_body: bool = False) -> List[multilspy_types.UnifiedSymbolInformation]:
         """
         Will go through all files in the project and build a tree of symbols. Note: this may be slow the first time it is called.
         
@@ -811,10 +812,10 @@ class LanguageServer:
 
                     _, root_nodes = await self.request_document_symbols(item_path, include_body=include_body)
                     
-                    # Create module symbol
-                    module_symbol = multilspy_types.UnifiedSymbolInformation( # type: ignore
+                    # Create file symbol
+                    file_symbol = multilspy_types.UnifiedSymbolInformation( # type: ignore
                         name=os.path.splitext(item)[0],
-                        kind=multilspy_types.SymbolKind.Module,
+                        kind=multilspy_types.SymbolKind.File,
                         location=multilspy_types.Location(
                             uri=str(pathlib.Path(abs_item_path).as_uri()),
                             range={"start": {"line": 0, "character": 0}, "end": {"line": 0, "character": 0}},
@@ -824,14 +825,57 @@ class LanguageServer:
                         children=root_nodes
                     )
                     
-                    package_symbol["children"].append(module_symbol)
+                    package_symbol["children"].append(file_symbol)
 
             return result
 
         # Start from the root or the specified directory
-        start_path = start_package_relative_path or self.repository_root_path
+        start_path = start_dir_relative_path or self.repository_root_path
         return await process_directory(start_path)
+    
+    async def request_dir_overview(self, relative_dir_path: str) -> dict[str, list[tuple[str, multilspy_types.SymbolKind, int, int]]]:
+        """
+        An overview of the given directory. 
+        
+        Maps relative paths of all contained files to info about top-level symbols in the file 
+        (name, kind, line, column).
+        """
+        if not self.server_started:
+            self.logger.log(
+                "request_dir_overview called before Language Server started",
+                logging.ERROR,
+            )
+            raise MultilspyException("Language Server not started")
 
+        # Get the full symbol tree for the directory
+        symbol_tree = await self.request_full_symbol_tree(relative_dir_path)
+        
+        # Initialize result dictionary
+        result: dict[str, list[tuple[str, multilspy_types.SymbolKind, int, int]]] = defaultdict(list)
+        
+        # Helper function to process a symbol and its children
+        def process_symbol(symbol: multilspy_types.UnifiedSymbolInformation):
+            if symbol["kind"] == multilspy_types.SymbolKind.File:
+                # For file symbols, process their children (top-level symbols)
+                for child in symbol["children"]:
+                    assert "location" in child
+                    assert "selectionRange" in child
+                    path = Path(child["location"]["relativePath"]).resolve().relative_to(self.repository_root_path)
+                    result[str(path)].append((
+                        child["name"],
+                        child["kind"],
+                        child["selectionRange"]["start"]["line"],
+                        child["selectionRange"]["start"]["character"]
+                    ))
+            # For package/directory symbols, process their children
+            for child in symbol["children"]:
+                process_symbol(child)
+        
+        # Process each root symbol
+        for root in symbol_tree:
+            process_symbol(root)
+        return result
+    
     async def request_hover(self, relative_file_path: str, line: int, column: int) -> Union[multilspy_types.Hover, None]:
         """
         Raise a [textDocument/hover](https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_hover) request to the Language Server
@@ -907,7 +951,7 @@ class LanguageServer:
         roots = await self.request_full_symbol_tree()
         paths = [] 
         def collect_module_files(symbol):
-            if symbol["kind"] == multilspy_types.SymbolKind.Module:
+            if symbol["kind"] == multilspy_types.SymbolKind.File:
                 assert "location" in symbol
                 paths.append(symbol["location"]["relativePath"])
             
@@ -946,6 +990,7 @@ class LanguageServer:
         all_files = await self.request_parsed_files()
         for path in all_files:
             # Apply glob filters if provided
+            # TODO: fnmatch is not exactly the same as glob
             if paths_include_glob and not fnmatch(path, paths_include_glob):
                 self.logger.log(f"Skipping {path}: does not match include pattern {paths_include_glob}", logging.DEBUG)
                 continue
@@ -1467,7 +1512,19 @@ class SyncLanguageServer:
             self.language_server.request_full_symbol_tree(start_package_relative_path, include_body), self.loop
         ).result()
         return result
-
+    
+    def request_dir_overview(self, relative_dir_path: str) -> dict[str, list[tuple[str, multilspy_types.SymbolKind, int, int]]]:
+        """
+        An overview of the given directory. 
+        
+        Maps relative paths of all contained files to info about top-level symbols in the file 
+        (name, kind, line, column).
+        """
+        assert self.loop
+        result = asyncio.run_coroutine_threadsafe(
+            self.language_server.request_dir_overview(relative_dir_path), self.loop
+        ).result()
+        return result
 
     def request_hover(self, relative_file_path: str, line: int, column: int) -> Union[multilspy_types.Hover, None]:
         """

src/multilspy/lsp_protocol_handler/lsp_types.py

@@ -191,6 +191,13 @@ class SymbolKind(IntEnum):
     Event = 24
     Operator = 25
     TypeParameter = 26
+    
+    @classmethod    
+    def from_int(cls, value: int) -> "SymbolKind":
+        for symbol_kind in cls:
+            if symbol_kind.value == value:
+                return symbol_kind
+        raise ValueError(f"Invalid symbol kind: {value}")
 
 
 class SymbolTag(IntEnum):

src/serena/mcp.py

@@ -186,6 +186,25 @@ def list_dir(ctx: Context, relative_path: str, recursive: bool) -> str:
     return ListDirTool(ctx).execute()
 
 
+@mcp.tool()
+def get_dir_overview(ctx: Context, relative_path: str) -> str:
+    """
+    Get an overview of the given directory.
+    For each file in the directory, we list the top-level symbols in the file (name, kind, line).
+
+    :param ctx: the context object, which will be created and provided automatically
+    :param relative_path: the relative path to the directory to get the overview of
+    :return: a JSON object mapping relative paths of all contained files to info about top-level symbols in the file (name, kind, line).
+    """
+    log.info(f"get_dir_overview: {relative_path=}")
+
+    class GetDirOverviewTool(Tool):
+        def _execute(self) -> str:
+            return json.dumps(self.langsrv.request_dir_overview(relative_path))
+
+    return GetDirOverviewTool(ctx).execute()
+
+
 @mcp.tool()
 def find_symbol(ctx: Context, name: str, depth: int = 0) -> str:
     """

test/multilspy/test_symbol_retrieval.py

@@ -6,6 +6,7 @@ These tests focus on the following methods:
 - request_referencing_symbols
 """
 
+import os
 from pathlib import Path
 
 from multilspy.language_server import SyncLanguageServer
@@ -26,7 +27,8 @@ class TestLanguageServerSymbols:
         assert containing_symbol is not None
         assert containing_symbol["name"] == "create_user"
         assert containing_symbol["kind"] == SymbolKind.Method
-        assert containing_symbol["body"].strip().startswith("def create_user(self")
+        if "body" in containing_symbol:
+            assert containing_symbol["body"].strip().startswith("def create_user(self")
 
     def test_references_to_variables(self, language_server: SyncLanguageServer, repo_path: Path):
         """Test request_referencing_symbols for a variable."""
@@ -35,7 +37,7 @@ class TestLanguageServerSymbols:
         ref_symbols = language_server.request_referencing_symbols(file_path, 74, 4)
 
         assert len(ref_symbols) > 0
-        ref_lines = [ref["range"]["start"]["line"] for ref in ref_symbols]
+        ref_lines = [ref["location"]["range"]["start"]["line"] for ref in ref_symbols if "location" in ref and "range" in ref["location"]]
         ref_names = [ref["name"] for ref in ref_symbols]
         assert 87 in ref_lines
         assert 95 in ref_lines
@@ -67,14 +69,17 @@ class TestLanguageServerSymbols:
         assert containing_symbol["kind"] == SymbolKind.Method
 
         # Get the parent containing symbol
-        parent_symbol = language_server.request_containing_symbol(
-            file_path, containing_symbol["range"]["start"]["line"], containing_symbol["range"]["start"]["character"] - 1
-        )
+        if "location" in containing_symbol and "range" in containing_symbol["location"]:
+            parent_symbol = language_server.request_containing_symbol(
+                file_path,
+                containing_symbol["location"]["range"]["start"]["line"],
+                containing_symbol["location"]["range"]["start"]["character"] - 1,
+            )
 
-        # Verify that the parent is the class
-        assert parent_symbol is not None
-        assert parent_symbol["name"] == "UserService"
-        assert parent_symbol["kind"] == SymbolKind.Class
+            # Verify that the parent is the class
+            assert parent_symbol is not None
+            assert parent_symbol["name"] == "UserService"
+            assert parent_symbol["kind"] == SymbolKind.Class
 
     def test_request_containing_symbol_none(self, language_server: SyncLanguageServer, repo_path: Path):
         """Test request_containing_symbol for a position with no containing symbol."""
@@ -100,7 +105,9 @@ class TestLanguageServerSymbols:
         for symbol in ref_symbols:
             assert "name" in symbol
             assert "kind" in symbol
-            assert "range" in symbol
+            if "location" in symbol and "range" in symbol["location"]:
+                assert "start" in symbol["location"]["range"]
+                assert "end" in symbol["location"]["range"]
 
     def test_request_referencing_symbols_class(self, language_server: SyncLanguageServer, repo_path: Path):
         """Test request_referencing_symbols for a class."""
@@ -113,7 +120,11 @@ class TestLanguageServerSymbols:
         assert len(ref_symbols) > 0
 
         # At least one reference should be from services.py
-        services_references = [symbol for symbol in ref_symbols if "services.py" in symbol.get("location", {}).get("uri", "")]
+        services_references = [
+            symbol
+            for symbol in ref_symbols
+            if "location" in symbol and "uri" in symbol["location"] and "services.py" in symbol["location"]["uri"]
+        ]
 
         assert len(services_references) > 0
 
@@ -128,7 +139,11 @@ class TestLanguageServerSymbols:
         assert len(ref_symbols) > 0
 
         # Verify the symbols include references within the method body
-        method_refs = [symbol for symbol in ref_symbols if symbol.get("location", {}).get("range", {}).get("start", {}).get("line", 0) > 24]
+        method_refs = [
+            symbol
+            for symbol in ref_symbols
+            if "location" in symbol and "range" in symbol["location"] and symbol["location"]["range"]["start"]["line"] > 24
+        ]
 
         assert len(method_refs) > 0
 
@@ -164,7 +179,8 @@ class TestLanguageServerSymbols:
         # Verify the location and kind of the symbol
         # SymbolKind.Method = 6 for a method
         assert defining_symbol.get("kind") == SymbolKind.Method.value
-        assert "services.py" in defining_symbol.get("location", {}).get("uri", "")
+        if "location" in defining_symbol and "uri" in defining_symbol["location"]:
+            assert "services.py" in defining_symbol["location"]["uri"]
 
     def test_request_defining_symbol_imported_class(self, language_server: SyncLanguageServer, repo_path: Path):
         """Test request_defining_symbol for an imported class."""
@@ -192,7 +208,8 @@ class TestLanguageServerSymbols:
             assert defining_symbol is not None
             assert defining_symbol.get("name") == "create_user"
             # The defining symbol should be in the services.py file
-            assert "services.py" in defining_symbol.get("location", {}).get("uri", "")
+            if "location" in defining_symbol and "uri" in defining_symbol["location"]:
+                assert "services.py" in defining_symbol["location"]["uri"]
         except AssertionError:
             # If the file structure doesn't match what we expect, we can't guarantee this test
             # will pass, so we'll consider it a warning rather than a failure
@@ -293,9 +310,8 @@ class TestLanguageServerSymbols:
 
         # Step 3: Verify that they refer to the same symbol
         assert defining_symbol["kind"] == containing_symbol["kind"]
-        assert "location" in defining_symbol
-        assert "location" in containing_symbol
-        assert defining_symbol["location"]["uri"] == containing_symbol["location"]["uri"]
+        if "location" in defining_symbol and "location" in containing_symbol:
+            assert defining_symbol["location"]["uri"] == containing_symbol["location"]["uri"]
 
         # The integration test is successful if we've gotten this far,
         # as it demonstrates the integration between request_containing_symbol and request_defining_symbol
@@ -338,14 +354,15 @@ class TestLanguageServerSymbols:
         examples_package = next(child for child in repo_structure[0]["children"] if child["name"] == "examples")
         # assert that children are __init__ and user_management
         assert {child["name"] for child in examples_package["children"]} == {"__init__", "user_management"}
-        assert {child["kind"] for child in examples_package["children"]} == {SymbolKind.Module}
+        assert {child["kind"] for child in examples_package["children"]} == {SymbolKind.File}
 
         # assert that tree of user_management node is same as retrieved directly
         user_management_node = next(child for child in examples_package["children"] if child["name"] == "user_management")
-        user_management_rel_path = user_management_node["location"]["relativePath"]
-        assert user_management_rel_path == "examples/user_management.py"
-        _, user_management_roots = language_server.request_document_symbols(str(repo_path / "examples" / "user_management.py"))
-        assert user_management_roots == user_management_node["children"]
+        if "location" in user_management_node and "relativePath" in user_management_node["location"]:
+            user_management_rel_path = user_management_node["location"]["relativePath"]
+            assert user_management_rel_path == "examples/user_management.py"
+            _, user_management_roots = language_server.request_document_symbols(str(repo_path / "examples" / "user_management.py"))
+            assert user_management_roots == user_management_node["children"]
 
     def test_symbol_tree_structure_subdir(self, language_server: SyncLanguageServer, repo_path: Path):
         """Test that the symbol tree structure is correctly built."""
@@ -357,11 +374,36 @@ class TestLanguageServerSymbols:
         assert examples_package["kind"] == SymbolKind.Package
         # assert that children are __init__ and user_management
         assert {child["name"] for child in examples_package["children"]} == {"__init__", "user_management"}
-        assert {child["kind"] for child in examples_package["children"]} == {SymbolKind.Module}
+        assert {child["kind"] for child in examples_package["children"]} == {SymbolKind.File}
 
         # assert that tree of user_management node is same as retrieved directly
         user_management_node = next(child for child in examples_package["children"] if child["name"] == "user_management")
-        user_management_rel_path = user_management_node["location"]["relativePath"]
-        assert user_management_rel_path == "examples/user_management.py"
-        _, user_management_roots = language_server.request_document_symbols(str(repo_path / "examples" / "user_management.py"))
-        assert user_management_roots == user_management_node["children"]
+        if "location" in user_management_node and "relativePath" in user_management_node["location"]:
+            user_management_rel_path = user_management_node["location"]["relativePath"]
+            assert user_management_rel_path == "examples/user_management.py"
+            _, user_management_roots = language_server.request_document_symbols(str(repo_path / "examples" / "user_management.py"))
+            assert user_management_roots == user_management_node["children"]
+
+    def test_request_dir_overview(self, language_server: SyncLanguageServer, repo_path: Path):
+        """Test that request_dir_overview returns correct symbol information for files in a directory."""
+        # Get overview of the examples directory
+        overview = language_server.request_dir_overview(str(repo_path / "test_repo"))
+
+        # Verify that we have entries for both files
+        assert os.path.join("test_repo", "nested.py") in overview
+
+        # Get the symbols for user_management.py
+        services_symbols = overview[os.path.join("test_repo", "services.py")]
+        assert len(services_symbols) > 0
+
+        # Check for specific symbols from services.py
+        expected_symbols = [
+            ("UserService", SymbolKind.Class, 9, 6),
+            ("ItemService", SymbolKind.Class, 40, 6),
+            ("create_service_container", SymbolKind.Function, 67, 4),
+            ("user_var_str", SymbolKind.Variable, 73, 0),
+            ("user_service", SymbolKind.Variable, 76, 0),
+        ]
+
+        for symbol in expected_symbols:
+            assert symbol in services_symbols