From 78b43d2a631a7626f020a484cb7e8455cbaf88d5 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Mon, 7 Apr 2025 00:54:34 +0200 Subject: [PATCH 01/12] LS: support for configuring ignored paths Introduced per-language always ignored paths --- .vscode/settings.json | 1 + src/multilspy/language_server.py | 99 ++++++++++++++----- .../eclipse_jdtls/eclipse_jdtls.py | 20 ++++ src/multilspy/language_servers/gopls/gopls.py | 10 ++ .../jedi_language_server/jedi_server.py | 8 +- .../language_servers/omnisharp/omnisharp.py | 8 +- .../pyright_language_server/pyright_server.py | 9 +- .../rust_analyzer/rust_analyzer.py | 6 ++ .../language_servers/solargraph/solargraph.py | 6 +- .../typescript_language_server.py | 13 ++- src/multilspy/multilspy_config.py | 4 +- .../test_retrieval_with_ignored_dirs.py | 46 +++++++++ test/resources/test_repo/scripts/run_app.py | 2 + 13 files changed, 203 insertions(+), 29 deletions(-) create mode 100644 test/multilspy/test_retrieval_with_ignored_dirs.py diff --git a/.vscode/settings.json b/.vscode/settings.json index 93348cb8..8a290211 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,6 +3,7 @@ "agno", "asyncio", "genai", + "Gopls", "langsrv", "multilspy", "pixi", diff --git a/src/multilspy/language_server.py b/src/multilspy/language_server.py index ae5fa594..9af94fb3 100644 --- a/src/multilspy/language_server.py +++ b/src/multilspy/language_server.py @@ -16,8 +16,10 @@ import pickle import re import threading from collections import defaultdict +from urllib.parse import urlparse from contextlib import asynccontextmanager, contextmanager from copy import copy +from typing import Any, AsyncGenerator, Dict, Generator, List, Optional, Set, Tuple, Union from fnmatch import fnmatch from pathlib import Path, PurePath from typing import AsyncIterator, Dict, Iterator, List, Optional, Tuple, Union, cast @@ -78,6 +80,12 @@ class LSPFileBuffer: class LanguageServer: + + # To be overridden and extended by subclasses + def should_always_ignore(self, dirname: str) -> bool: + return dirname.startswith('.') + + """ The LanguageServer class provides a language agnostic interface to the Language Server Protocol. It is used to communicate with Language Servers of different programming languages. @@ -172,6 +180,14 @@ class LanguageServer: self.server_started = False self.repository_root_path: str = repository_root_path self.completions_available = asyncio.Event() + + # for all absolute paths in ignored_dirs, convert them to relative paths + self.ignored_paths = [] + for path in set(config.ignored_paths): + if os.path.isabs(path): + path = str(Path(path).resolve().relative_to(self.repository_root_path)) + self.ignored_paths.append(path) + if config.trace_lsp_communication: @@ -196,6 +212,27 @@ class LanguageServer: self.load_cache() self._cache_has_changed = bool self.language = Language(language_id) + + # Helper function to check if a relative path should be ignored + def should_ignore_path(self, relative_path: str) -> bool: + fn_matcher = self.language.get_source_fn_matcher() + if os.path.isfile(relative_path) and not fn_matcher.is_relevant_filename(relative_path): + return True + + # Normalize path separators for consistent comparison + rel_path = Path(relative_path) + for ignored_path in self.ignored_paths: + if rel_path.is_relative_to(ignored_path): + return True + + # Check each part of the path against always fulfilled ignore conditions + for part in rel_path.parts: + if not part: # Skip empty parts (e.g., from leading '/') + continue + # Check standard ignores + if self.should_always_ignore(part): + return True + return False @asynccontextmanager async def start_server(self) -> AsyncIterator["LanguageServer"]: @@ -463,21 +500,23 @@ class LanguageServer: """ Raise a [textDocument/references](https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_references) request to the Language Server to find references to the symbol at the given line and column in the given file. Wait for the response and return the result. + Filters out references located in ignored directories. :param relative_file_path: The relative path of the file that has the symbol for which references should be looked up :param line: The line number of the symbol :param column: The column number of the symbol - :return List[multilspy_types.Location]: A list of locations where the symbol is referenced + :return: A list of locations where the symbol is referenced (excluding ignored directories) """ if not self.server_started: self.logger.log( - "find_all_callers_of_function called before Language Server started", + "request_references called before Language Server started", logging.ERROR, ) raise MultilspyException("Language Server not started") + with self.open_file(relative_file_path): # sending request to the language server and waiting for response response = await self.server.send.references( @@ -496,13 +535,17 @@ class LanguageServer: assert isinstance(item, dict) assert LSPConstants.URI in item assert LSPConstants.RANGE in item + + abs_path = PathUtils.uri_to_path(item[LSPConstants.URI]) + rel_path = Path(abs_path).relative_to(self.repository_root_path) + if self.should_ignore_path(str(rel_path)): + self.logger.log(f"Ignoring reference in {rel_path} since it should be ignored", logging.DEBUG) + continue new_item: multilspy_types.Location = {} new_item.update(item) - new_item["absolutePath"] = PathUtils.uri_to_path(new_item["uri"]) - new_item["relativePath"] = str( - PurePath(os.path.relpath(new_item["absolutePath"], self.repository_root_path)) - ) + new_item["absolutePath"] = str(abs_path) + new_item["relativePath"] = str(rel_path) ret.append(multilspy_types.Location(**new_item)) return ret @@ -749,12 +792,13 @@ class LanguageServer: async def request_full_symbol_tree(self, within_relative_path: str | None = None, include_body: bool = False) -> List[multilspy_types.UnifiedSymbolInformation]: """ Will go through all files in the project and build a tree of symbols. Note: this may be slow the first time it is called. - + For each file, a symbol of kind Module (3) will be created. For directories, a symbol of kind Package (4) will be created. - All symbols will have a children attribute, thereby representing the tree structure of all symbols in the project + All symbols will have a children attribute, thereby representing the tree structure of all symbols in the project that are within the repository. - Will ignore all directories that start with a dot (.) and __pycache__ directories. - + Will ignore directories starting with '.', '__pycache__', language-specific defaults (self.default_ignored_dirs), + and user-configured directories (self.config.ignored_dirs). + :param within_relative_path: pass a relative path to only consider symbols within this path. If a file is passed, only the symbols within this file will be considered. If a directory is passed, all files within this directory will be considered. @@ -763,22 +807,20 @@ class LanguageServer: :return: A list of root symbols representing the top-level packages/modules in the project. """ if within_relative_path is not None and os.path.isfile(within_relative_path): + if self.should_ignore_path(within_relative_path): + self.logger.log(f"You passed a file explicitly, but it is ignored. This is probably an error. File: {within_relative_path}", logging.ERROR) + return [] + _, root_nodes = await self.request_document_symbols(within_relative_path, include_body=include_body) return root_nodes - - # Helper function to check if a path should be ignored - def should_ignore_dir(path: str) -> bool: - parts = path.split(os.sep) - return any(part.startswith('.') or part == '__pycache__' for part in parts) - - fn_matcher = self.language.get_source_fn_matcher() - + # Helper function to recursively process directories async def process_directory(dir_path: str) -> List[multilspy_types.UnifiedSymbolInformation]: abs_dir_path = self.repository_root_path if dir_path == "." else os.path.join(self.repository_root_path, dir_path) abs_dir_path = os.path.realpath(abs_dir_path) - if should_ignore_dir(abs_dir_path): + if self.should_ignore_path(str(Path(abs_dir_path).relative_to(self.repository_root_path))): + self.logger.log(f"Skipping directory: {dir_path}\n(because it should be ignored)", logging.DEBUG) return [] result = [] @@ -804,15 +846,16 @@ class LanguageServer: for item in items: item_path = os.path.join(abs_dir_path, item) abs_item_path = os.path.join(self.repository_root_path, item_path) + rel_item_path = str(Path(abs_item_path).resolve().relative_to(self.repository_root_path)) + if self.should_ignore_path(rel_item_path): + self.logger.log(f"Skipping item: {rel_item_path}\n(because it should be ignored)", logging.DEBUG) + continue if os.path.isdir(abs_item_path): child_symbols = await process_directory(item_path) package_symbol["children"].extend(child_symbols) elif os.path.isfile(abs_item_path): - if not fn_matcher.is_relevant_filename(item): - continue - _, root_nodes = await self.request_document_symbols(item_path, include_body=include_body) def fix_relative_path(nodes: List[multilspy_types.UnifiedSymbolInformation]): @@ -1851,3 +1894,15 @@ class SyncLanguageServer: Load the cache from a file. """ self.language_server.load_cache() + + def should_always_ignore(self, dirname: str) -> bool: + """ + Whether the given directory should be ignored. + """ + return self.language_server.should_always_ignore(dirname) + + def should_ignore_path(self, relative_path: str) -> bool: + """ + Whether the given path should be ignored. + """ + return self.language_server.should_ignore_path(relative_path) diff --git a/src/multilspy/language_servers/eclipse_jdtls/eclipse_jdtls.py b/src/multilspy/language_servers/eclipse_jdtls/eclipse_jdtls.py index 247f0558..e71b3042 100644 --- a/src/multilspy/language_servers/eclipse_jdtls/eclipse_jdtls.py +++ b/src/multilspy/language_servers/eclipse_jdtls/eclipse_jdtls.py @@ -14,6 +14,8 @@ import uuid from contextlib import asynccontextmanager from typing import AsyncIterator +from overrides import override + from multilspy.multilspy_logger import MultilspyLogger from multilspy.language_server import LanguageServer from multilspy.lsp_protocol_handler.server import ProcessLaunchInfo @@ -138,6 +140,24 @@ class EclipseJDTLS(LanguageServer): self.initialize_searcher_command_available = asyncio.Event() super().__init__(config, logger, repository_root_path, ProcessLaunchInfo(cmd, proc_env, proc_cwd), "java") + + @override + def should_always_ignore(self, dirname: str) -> bool: + # Ignore common Java build directories from different build tools: + # - Maven: target + # - Gradle: build, .gradle + # - Eclipse: bin, .settings + # - IntelliJ IDEA: out, .idea + # - General: classes, dist, lib + return super().should_always_ignore(dirname) or dirname in [ + "target", # Maven + "build", # Gradle + "bin", # Eclipse + "out", # IntelliJ IDEA + "classes", # General + "dist", # General + "lib" # General + ] def setupRuntimeDependencies(self, logger: MultilspyLogger, config: MultilspyConfig) -> RuntimeDependencyPaths: """ diff --git a/src/multilspy/language_servers/gopls/gopls.py b/src/multilspy/language_servers/gopls/gopls.py index 40031db4..89a16e5d 100644 --- a/src/multilspy/language_servers/gopls/gopls.py +++ b/src/multilspy/language_servers/gopls/gopls.py @@ -7,6 +7,8 @@ import subprocess from contextlib import asynccontextmanager from typing import AsyncIterator +from overrides import override + from multilspy.multilspy_logger import MultilspyLogger from multilspy.language_server import LanguageServer from multilspy.lsp_protocol_handler.server import ProcessLaunchInfo @@ -18,6 +20,14 @@ class Gopls(LanguageServer): """ Provides Go specific instantiation of the LanguageServer class using gopls. """ + + @override + def should_always_ignore(self, dirname: str) -> bool: + # For Go projects, we should ignore: + # - vendor: third-party dependencies vendored into the project + # - node_modules: if the project has JavaScript components + # - dist/build: common output directories + return super().should_always_ignore(dirname) or dirname in ["vendor", "node_modules", "dist", "build"] @staticmethod def _get_go_version(): diff --git a/src/multilspy/language_servers/jedi_language_server/jedi_server.py b/src/multilspy/language_servers/jedi_language_server/jedi_server.py index d49ace2c..1f1dc883 100644 --- a/src/multilspy/language_servers/jedi_language_server/jedi_server.py +++ b/src/multilspy/language_servers/jedi_language_server/jedi_server.py @@ -7,7 +7,9 @@ import logging import os import pathlib from contextlib import asynccontextmanager -from typing import AsyncIterator +from typing import AsyncIterator, Tuple + +from overrides import override from multilspy.multilspy_logger import MultilspyLogger from multilspy.language_server import LanguageServer @@ -32,6 +34,10 @@ class JediServer(LanguageServer): ProcessLaunchInfo(cmd="jedi-language-server", cwd=repository_root_path), "python", ) + + @override + def should_always_ignore(self, dirname: str) -> bool: + return super().should_always_ignore(dirname) or dirname in ["venv", "__pycache__"] def _get_initialize_params(self, repository_absolute_path: str) -> InitializeParams: """ diff --git a/src/multilspy/language_servers/omnisharp/omnisharp.py b/src/multilspy/language_servers/omnisharp/omnisharp.py index d290af2a..21174b29 100644 --- a/src/multilspy/language_servers/omnisharp/omnisharp.py +++ b/src/multilspy/language_servers/omnisharp/omnisharp.py @@ -11,6 +11,8 @@ import stat from contextlib import asynccontextmanager from typing import AsyncIterator, Iterable +from overrides import override + from multilspy.multilspy_logger import MultilspyLogger from multilspy.language_server import LanguageServer from multilspy.lsp_protocol_handler.server import ProcessLaunchInfo @@ -46,7 +48,7 @@ def breadth_first_file_scan(root) -> Iterable[str]: dirs = next_dirs -def find_least_depth_sln_file(root_dir) -> str: +def find_least_depth_sln_file(root_dir) -> str | None: for filename in breadth_first_file_scan(root_dir): if filename.endswith(".sln"): return filename @@ -107,6 +109,10 @@ class OmniSharp(LanguageServer): self.definition_available = asyncio.Event() self.references_available = asyncio.Event() + + @override + def should_always_ignore(self, dirname: str) -> bool: + return super().should_always_ignore(dirname) or dirname in ["bin", "obj"] def _get_initialize_params(self, repository_absolute_path: str) -> InitializeParams: """ diff --git a/src/multilspy/language_servers/pyright_language_server/pyright_server.py b/src/multilspy/language_servers/pyright_language_server/pyright_server.py index 09e301f3..5cffb059 100644 --- a/src/multilspy/language_servers/pyright_language_server/pyright_server.py +++ b/src/multilspy/language_servers/pyright_language_server/pyright_server.py @@ -7,7 +7,9 @@ import logging import os import pathlib from contextlib import asynccontextmanager -from typing import AsyncIterator +from typing import AsyncIterator, Tuple + +from overrides import override from multilspy.multilspy_logger import MultilspyLogger from multilspy.language_server import LanguageServer @@ -21,7 +23,6 @@ class PyrightServer(LanguageServer): Provides Python specific instantiation of the LanguageServer class using Pyright. Contains various configurations and settings specific to Python. """ - def __init__(self, config: MultilspyConfig, logger: MultilspyLogger, repository_root_path: str): """ Creates a PyrightServer instance. This class is not meant to be instantiated directly. @@ -36,6 +37,10 @@ class PyrightServer(LanguageServer): ProcessLaunchInfo(cmd="python -m pyright.langserver --stdio", cwd=repository_root_path), "python", ) + + @override + def should_always_ignore(self, dirname: str) -> bool: + return super().should_always_ignore(dirname) or dirname in ["venv", "__pycache__"] def _get_initialize_params(self, repository_absolute_path: str) -> InitializeParams: """ diff --git a/src/multilspy/language_servers/rust_analyzer/rust_analyzer.py b/src/multilspy/language_servers/rust_analyzer/rust_analyzer.py index 1a95eec6..0acdddf5 100644 --- a/src/multilspy/language_servers/rust_analyzer/rust_analyzer.py +++ b/src/multilspy/language_servers/rust_analyzer/rust_analyzer.py @@ -11,6 +11,8 @@ import pathlib from contextlib import asynccontextmanager from typing import AsyncIterator +from overrides import override + from multilspy.multilspy_logger import MultilspyLogger from multilspy.language_server import LanguageServer from multilspy.lsp_protocol_handler.server import ProcessLaunchInfo @@ -38,6 +40,10 @@ class RustAnalyzer(LanguageServer): "rust", ) self.server_ready = asyncio.Event() + + @override + def should_always_ignore(self, dirname: str) -> bool: + return super().should_always_ignore(dirname) or dirname in ["target"] def setup_runtime_dependencies(self, logger: MultilspyLogger, config: MultilspyConfig) -> str: """ diff --git a/src/multilspy/language_servers/solargraph/solargraph.py b/src/multilspy/language_servers/solargraph/solargraph.py index a0a62b8f..4dadb735 100644 --- a/src/multilspy/language_servers/solargraph/solargraph.py +++ b/src/multilspy/language_servers/solargraph/solargraph.py @@ -11,7 +11,7 @@ import stat import subprocess import pathlib from contextlib import asynccontextmanager -from typing import AsyncIterator +from typing import AsyncIterator, override from multilspy.multilspy_logger import MultilspyLogger from multilspy.language_server import LanguageServer @@ -42,6 +42,10 @@ class Solargraph(LanguageServer): "ruby", ) self.server_ready = asyncio.Event() + + @override + def should_always_ignore(self, dirname: str) -> bool: + return super().should_always_ignore(dirname) or dirname in ["vendor"] def setup_runtime_dependencies(self, logger: MultilspyLogger, config: MultilspyConfig, repository_root_path: str) -> str: """ diff --git a/src/multilspy/language_servers/typescript_language_server/typescript_language_server.py b/src/multilspy/language_servers/typescript_language_server/typescript_language_server.py index eb823ae2..85b1a18e 100644 --- a/src/multilspy/language_servers/typescript_language_server/typescript_language_server.py +++ b/src/multilspy/language_servers/typescript_language_server/typescript_language_server.py @@ -10,7 +10,9 @@ import os import subprocess import pathlib from contextlib import asynccontextmanager -from typing import AsyncIterator +from typing import AsyncIterator, Tuple + +from overrides import override from multilspy.multilspy_logger import MultilspyLogger from multilspy.language_server import LanguageServer @@ -48,6 +50,15 @@ class TypeScriptLanguageServer(LanguageServer): "typescript", ) self.server_ready = asyncio.Event() + + @override + def should_always_ignore(self, dirname: str) -> bool: + return super().should_always_ignore(dirname) or dirname in [ + "node_modules", + "dist", + "build", + "coverage", + ] def setup_runtime_dependencies(self, logger: MultilspyLogger, config: MultilspyConfig) -> str: """ diff --git a/src/multilspy/multilspy_config.py b/src/multilspy/multilspy_config.py index ac2ff976..db18a209 100644 --- a/src/multilspy/multilspy_config.py +++ b/src/multilspy/multilspy_config.py @@ -3,7 +3,8 @@ Configuration parameters for Multilspy. """ import fnmatch from enum import Enum -from dataclasses import dataclass +from typing import List +from dataclasses import dataclass, field class FilenameMatcher: @@ -66,6 +67,7 @@ class MultilspyConfig: """ code_language: Language trace_lsp_communication: bool = False + ignored_paths: List[str] = field(default_factory=list) @classmethod def from_dict(cls, env: dict): diff --git a/test/multilspy/test_retrieval_with_ignored_dirs.py b/test/multilspy/test_retrieval_with_ignored_dirs.py new file mode 100644 index 00000000..9d39f83a --- /dev/null +++ b/test/multilspy/test_retrieval_with_ignored_dirs.py @@ -0,0 +1,46 @@ +from collections.abc import Generator +from pathlib import Path + +import pytest + +from multilspy.language_server import SyncLanguageServer +from multilspy.multilspy_config import Language, MultilspyConfig +from multilspy.multilspy_logger import MultilspyLogger + + +@pytest.fixture(scope="module") +def ls_with_ignored_dirs(repo_path: Path) -> Generator[SyncLanguageServer, None, None]: + """Fixture to set up an LS for the python test repo with the 'scripts' directory ignored.""" + config = MultilspyConfig( + code_language=Language.PYTHON, + trace_lsp_communication=False, + ignored_paths=["scripts", "custom_test"], # Configure the relative path to be ignored + ) + logger = MultilspyLogger() + ls = SyncLanguageServer.create(config, logger, str(repo_path)) + ls.start() + try: + yield ls + finally: + ls.stop() + + +def test_symbol_tree_ignores_dir(ls_with_ignored_dirs: SyncLanguageServer): + """Tests that request_full_symbol_tree ignores the configured directory.""" + root = ls_with_ignored_dirs.request_full_symbol_tree()[0] + root_children = root["children"] + children_names = {child["name"] for child in root_children} + assert children_names == {"test_repo", "examples"} + + +def test_find_references_ignores_dir(ls_with_ignored_dirs: SyncLanguageServer): + """Tests that find_references ignores the configured directory.""" + # Location of Item, which is referenced in scripts + definition_file = "test_repo/models.py" + definition_line = 56 + definition_col = 6 + + references = ls_with_ignored_dirs.request_references(definition_file, definition_line, definition_col) + + # assert that scripts does not appear in the references + assert not any("scripts" in ref["relativePath"] for ref in references) diff --git a/test/resources/test_repo/scripts/run_app.py b/test/resources/test_repo/scripts/run_app.py index 8e97f1cf..40c76c6d 100644 --- a/test/resources/test_repo/scripts/run_app.py +++ b/test/resources/test_repo/scripts/run_app.py @@ -158,5 +158,7 @@ def main(): logger.info("Application completed successfully") +item_reference = Item(id="1", name="Item 1", price=10.0, category="Electronics") + if __name__ == "__main__": main() From 8e68e2f4c3d01fb78b5cb444066c5bb208538081 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Mon, 7 Apr 2025 00:58:26 +0200 Subject: [PATCH 02/12] Start LS with ignored dirs config --- src/serena/agent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index 9ea4c03a..00ba9f31 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -94,7 +94,7 @@ class SerenaAgent: ) # create and start the language server instance - config = MultilspyConfig(code_language=self.language) + config = MultilspyConfig(code_language=self.language, ignored_paths=project_config.get("ignored_dirs", [])) logger = MultilspyLogger() self.language_server = SyncLanguageServer.create(config, logger, self.project_root) @@ -366,7 +366,7 @@ class ListDirTool(Tool): os.path.join(self.project_root, relative_path), relative_to=self.project_root, recursive=recursive, - ignored_dirs=self.project_config["ignored_dirs"], + ignored_dirs=self.project_config.get("ignored_dirs", []), ) result = json.dumps({"dirs": dirs, "files": files}) return self._limit_length(result, max_answer_chars) From c4384682f281c344b404f726349626b0e84fc2e3 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Mon, 7 Apr 2025 15:43:41 +0200 Subject: [PATCH 03/12] LS: support glob ignores using pathspec --- pyproject.toml | 1 + src/multilspy/language_server.py | 54 +++++++++++++------ src/multilspy/multilspy_utils.py | 5 ++ .../test_retrieval_with_ignored_dirs.py | 25 +++++++++ uv.lock | 2 + 5 files changed, 71 insertions(+), 16 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8b48ffa2..7a92262b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ dependencies = [ "pyyaml>=6.0.2", "jinja2>=3.1.6", "dotenv>=0.9.9", + "pathspec>=0.12.1", ] [project.scripts] diff --git a/src/multilspy/language_server.py b/src/multilspy/language_server.py index 9af94fb3..4f661f9e 100644 --- a/src/multilspy/language_server.py +++ b/src/multilspy/language_server.py @@ -16,14 +16,15 @@ import pickle import re import threading from collections import defaultdict -from urllib.parse import urlparse from contextlib import asynccontextmanager, contextmanager from copy import copy -from typing import Any, AsyncGenerator, Dict, Generator, List, Optional, Set, Tuple, Union +from typing import Dict, List, Optional, Tuple, Union from fnmatch import fnmatch from pathlib import Path, PurePath from typing import AsyncIterator, Dict, Iterator, List, Optional, Tuple, Union, cast +import pathspec + from serena.text_utils import LineType, MatchedConsecutiveLines, TextLine, search_text from . import multilspy_types from .lsp_protocol_handler import lsp_types as LSPTypes @@ -181,14 +182,6 @@ class LanguageServer: self.repository_root_path: str = repository_root_path self.completions_available = asyncio.Event() - # for all absolute paths in ignored_dirs, convert them to relative paths - self.ignored_paths = [] - for path in set(config.ignored_paths): - if os.path.isabs(path): - path = str(Path(path).resolve().relative_to(self.repository_root_path)) - self.ignored_paths.append(path) - - if config.trace_lsp_communication: def logging_fn(source, target, msg): @@ -213,27 +206,56 @@ class LanguageServer: self._cache_has_changed = bool self.language = Language(language_id) - # Helper function to check if a relative path should be ignored + # Set up the pathspec matcher for the ignored paths + # for all absolute paths in ignored_paths, convert them to relative paths + processed_patterns = [] + for pattern in set(config.ignored_paths): + # Normalize separators (pathspec expects forward slashes) + pattern = pattern.replace(os.path.sep, '/') + processed_patterns.append(pattern) + # Create a pathspec matcher from the processed patterns + self.ignore_spec = pathspec.PathSpec.from_lines( + pathspec.patterns.GitWildMatchPattern, + processed_patterns + ) + + def should_ignore_path(self, relative_path: str) -> bool: + """ + Determine if a path should be ignored based on file type + and ignore patterns. + + :param relative_path: Relative path to check + + :return: True if the path should be ignored, False otherwise + """ + # Check file extension if it's a file fn_matcher = self.language.get_source_fn_matcher() if os.path.isfile(relative_path) and not fn_matcher.is_relevant_filename(relative_path): return True - # Normalize path separators for consistent comparison + # Create normalized path for consistent handling rel_path = Path(relative_path) - for ignored_path in self.ignored_paths: - if rel_path.is_relative_to(ignored_path): - return True # Check each part of the path against always fulfilled ignore conditions for part in rel_path.parts: - if not part: # Skip empty parts (e.g., from leading '/') + if not part: # Skip empty parts (e.g., from leading '/') continue # Check standard ignores if self.should_always_ignore(part): return True + + # Use pathspec for gitignore-style pattern matching + # Normalize path separators for pathspec (it expects forward slashes) + normalized_path = str(rel_path).replace(os.path.sep, '/') + + # Use the pathspec matcher to check if the path matches any ignore pattern + if self.ignore_spec.match_file(normalized_path): + return True + return False + @asynccontextmanager async def start_server(self) -> AsyncIterator["LanguageServer"]: """ diff --git a/src/multilspy/multilspy_utils.py b/src/multilspy/multilspy_utils.py index f1dc8dc1..6e42aa55 100644 --- a/src/multilspy/multilspy_utils.py +++ b/src/multilspy/multilspy_utils.py @@ -88,6 +88,11 @@ class PathUtils: parsed = urlparse(uri) host = "{0}{0}{mnt}{0}".format(os.path.sep, mnt=parsed.netloc) return os.path.normpath(os.path.join(host, url2pathname(unquote(parsed.path)))) + + @staticmethod + def is_glob_pattern(pattern: str) -> bool: + """Check if a pattern contains glob-specific characters.""" + return any(c in pattern for c in '*?[]!') class FileUtils: """ diff --git a/test/multilspy/test_retrieval_with_ignored_dirs.py b/test/multilspy/test_retrieval_with_ignored_dirs.py index 9d39f83a..09e4f538 100644 --- a/test/multilspy/test_retrieval_with_ignored_dirs.py +++ b/test/multilspy/test_retrieval_with_ignored_dirs.py @@ -44,3 +44,28 @@ def test_find_references_ignores_dir(ls_with_ignored_dirs: SyncLanguageServer): # assert that scripts does not appear in the references assert not any("scripts" in ref["relativePath"] for ref in references) + + +def test_refs_and_symbols_with_glob_patterns(repo_path: Path) -> None: + """Tests that refs and symbols with glob patterns are ignored.""" + config = MultilspyConfig( + code_language=Language.PYTHON, + trace_lsp_communication=False, + ignored_paths=["*ipts", "custom_t*"], + ) + logger = MultilspyLogger() + ls = SyncLanguageServer.create(config, logger, str(repo_path)) + ls.start() + # same as in the above tests + root = ls.request_full_symbol_tree()[0] + root_children = root["children"] + children_names = {child["name"] for child in root_children} + assert children_names == {"test_repo", "examples"} + + # test that the refs and symbols with glob patterns are ignored + definition_file = "test_repo/models.py" + definition_line = 56 + definition_col = 6 + + references = ls.request_references(definition_file, definition_line, definition_col) + assert not any("scripts" in ref["relativePath"] for ref in references) diff --git a/uv.lock b/uv.lock index 2535003c..10c6be61 100644 --- a/uv.lock +++ b/uv.lock @@ -950,6 +950,7 @@ dependencies = [ { name = "jinja2" }, { name = "mcp" }, { name = "overrides" }, + { name = "pathspec" }, { name = "pydantic" }, { name = "pyright" }, { name = "python-dotenv" }, @@ -997,6 +998,7 @@ requires-dist = [ { name = "mcp", specifier = ">=1.5.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.4.1" }, { name = "overrides", specifier = ">=7.7.0,<8" }, + { name = "pathspec", specifier = ">=0.12.1" }, { name = "poethepoet", marker = "extra == 'dev'", specifier = ">=0.20.0" }, { name = "pydantic", specifier = ">=2.10.6" }, { name = "pyright", specifier = ">=1.1.396,<2" }, From 54588962b7188a512017411a7366099b785473c2 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Mon, 7 Apr 2025 16:44:25 +0200 Subject: [PATCH 04/12] LS: option to add_gitignore_content_to_config Defaults to true since it's probably always a good idea --- src/multilspy/language_server.py | 51 +++++-- src/multilspy/multilspy_config.py | 7 +- test/resources/test_repo/.gitignore | 1 + .../ignored_module.py | 141 ++++++++++++++++++ 4 files changed, 187 insertions(+), 13 deletions(-) create mode 100644 test/resources/test_repo/.gitignore create mode 100644 test/resources/test_repo/ignore_this_dir_with_postfix/ignored_module.py diff --git a/src/multilspy/language_server.py b/src/multilspy/language_server.py index 7b88b863..93392131 100644 --- a/src/multilspy/language_server.py +++ b/src/multilspy/language_server.py @@ -81,18 +81,17 @@ class LSPFileBuffer: class LanguageServer: - - # To be overridden and extended by subclasses - def should_always_ignore(self, dirname: str) -> bool: - return dirname.startswith('.') - - """ The LanguageServer class provides a language agnostic interface to the Language Server Protocol. It is used to communicate with Language Servers of different programming languages. """ + + # To be overridden and extended by subclasses + def should_always_ignore(self, dirname: str) -> bool: + return dirname.startswith('.') + @classmethod - def create(cls, config: MultilspyConfig, logger: MultilspyLogger, repository_root_path: str) -> "LanguageServer": + def create(cls, config: MultilspyConfig, logger: MultilspyLogger, repository_root_path: str, add_gitignore_content_to_config: bool = True) -> "LanguageServer": """ Creates a language specific LanguageServer instance based on the given configuration, and appropriate settings for the programming language. @@ -102,9 +101,29 @@ class LanguageServer: :param repository_root_path: The root path of the repository. :param config: The Multilspy configuration. :param logger: The logger to use. + :param add_gitignore_content_to_config: whether to add the content of the .gitignore file (if any found) to the config, so that + the paths ignored there are also ignored by the language server :return LanguageServer: A language specific LanguageServer instance. """ + config = copy(config) # prevent mutation + if add_gitignore_content_to_config: + gitignore_path = os.path.join(repository_root_path, ".gitignore") + if not os.path.exists(gitignore_path): + logger.log( + f"Should ignore all files in gitignore not not .gitignore found at {gitignore_path}. Skipping.", + logging.WARNING + ) + gitignore_file_content = None + else: + if config.gitignore_file_content is not None: + raise ValueError( + f"Asked to add gitignore content to the config for {repository_root_path=} but there already is a non-empty entry" + ) + with open(gitignore_path) as f: + gitignore_file_content = f.read() + config.gitignore_file_content = gitignore_file_content + if config.code_language == Language.PYTHON: from multilspy.language_servers.pyright_language_server.pyright_server import ( PyrightServer, @@ -167,7 +186,7 @@ class LanguageServer: :param config: The Multilspy configuration. :param logger: The logger to use. :param repository_root_path: The root path of the repository. - :param cmd: Each language server has a specific command used to start the server. + :param process_launch_info: Each language server has a specific command used to start the server. This parameter is the command to launch the language server process. The command must pass appropriate flags to the binary, so that it runs in the stdio mode, as opposed to HTTP, TCP modes supported by some language servers. @@ -213,13 +232,16 @@ class LanguageServer: # Normalize separators (pathspec expects forward slashes) pattern = pattern.replace(os.path.sep, '/') processed_patterns.append(pattern) + # Combine explicitly passed patterns with + if config.gitignore_file_content is not None: + processed_patterns.extend(config.gitignore_file_content.splitlines()) + # Create a pathspec matcher from the processed patterns self.ignore_spec = pathspec.PathSpec.from_lines( pathspec.patterns.GitWildMatchPattern, processed_patterns ) - def should_ignore_path(self, relative_path: str) -> bool: """ Determine if a path should be ignored based on file type @@ -249,6 +271,11 @@ class LanguageServer: # Normalize path separators for pathspec (it expects forward slashes) normalized_path = str(rel_path).replace(os.path.sep, '/') + # pathspec can't handle the matching of directories if they don't end with a slash! + # see https://github.com/cpburnz/python-pathspec/issues/89 + if os.path.isdir(os.path.join(self.repository_root_path, normalized_path)) and not normalized_path.endswith('/'): + normalized_path = normalized_path + '/' + # Use the pathspec matcher to check if the path matches any ignore pattern if self.ignore_spec.match_file(normalized_path): return True @@ -1500,7 +1527,7 @@ class SyncLanguageServer: @classmethod def create( - cls, config: MultilspyConfig, logger: MultilspyLogger, repository_root_path: str + cls, config: MultilspyConfig, logger: MultilspyLogger, repository_root_path: str, add_gitignore_content_to_config=True, ) -> "SyncLanguageServer": """ Creates a language specific LanguageServer instance based on the given configuration, and appropriate settings for the programming language. @@ -1510,10 +1537,12 @@ class SyncLanguageServer: :param repository_root_path: The root path of the repository. :param config: The Multilspy configuration. :param logger: The logger to use. + :param add_gitignore_content_to_config: whether to add the content of the .gitignore file (if any found) to the config, so that + the paths ignored there are also ignored by the language server :return SyncLanguageServer: A language specific LanguageServer instance. """ - return SyncLanguageServer(LanguageServer.create(config, logger, repository_root_path)) + return SyncLanguageServer(LanguageServer.create(config, logger, repository_root_path, add_gitignore_content_to_config=add_gitignore_content_to_config)) @contextmanager def open_file(self, relative_file_path: str) -> Iterator[LSPFileBuffer]: diff --git a/src/multilspy/multilspy_config.py b/src/multilspy/multilspy_config.py index db18a209..c78759c4 100644 --- a/src/multilspy/multilspy_config.py +++ b/src/multilspy/multilspy_config.py @@ -67,7 +67,10 @@ class MultilspyConfig: """ code_language: Language trace_lsp_communication: bool = False - ignored_paths: List[str] = field(default_factory=list) + ignored_paths: list[str] = field(default_factory=list) + """Paths, dirs or glob-like patterns. The matching will follow the same logic as for .gitignore entries""" + gitignore_file_content: str | None = None + """Optional content of the gitignore file. If passed, will be used in addition to the explicitly passed ignored_paths for deciding which paths to ignore.""" @classmethod def from_dict(cls, env: dict): @@ -78,4 +81,4 @@ class MultilspyConfig: return cls(**{ k: v for k, v in env.items() if k in inspect.signature(cls).parameters - }) \ No newline at end of file + }) diff --git a/test/resources/test_repo/.gitignore b/test/resources/test_repo/.gitignore new file mode 100644 index 00000000..811c627c --- /dev/null +++ b/test/resources/test_repo/.gitignore @@ -0,0 +1 @@ +ignore_this_dir*/ \ No newline at end of file diff --git a/test/resources/test_repo/ignore_this_dir_with_postfix/ignored_module.py b/test/resources/test_repo/ignore_this_dir_with_postfix/ignored_module.py new file mode 100644 index 00000000..1e6da936 --- /dev/null +++ b/test/resources/test_repo/ignore_this_dir_with_postfix/ignored_module.py @@ -0,0 +1,141 @@ +""" +Example demonstrating user management with the test_repo module. + +This example showcases: +- Creating and managing users +- Using various object types and relationships +- Type annotations and complex Python patterns +""" + +import logging +from dataclasses import dataclass +from typing import Any + +from test_repo.models import User, create_user_object +from test_repo.services import UserService + +# Set up logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +@dataclass +class UserStats: + """Statistics about user activity.""" + + user_id: str + login_count: int = 0 + last_active_days: int = 0 + engagement_score: float = 0.0 + + def is_active(self) -> bool: + """Check if the user is considered active.""" + return self.last_active_days < 30 + + +class UserManager: + """Example class demonstrating complex user management.""" + + def __init__(self, service: UserService): + self.service = service + self.active_users: dict[str, User] = {} + self.user_stats: dict[str, UserStats] = {} + + def register_user(self, name: str, email: str, roles: list[str] | None = None) -> User: + """Register a new user.""" + logger.info(f"Registering new user: {name} ({email})") + user = self.service.create_user(name=name, email=email, roles=roles) + self.active_users[user.id] = user + self.user_stats[user.id] = UserStats(user_id=user.id) + return user + + def get_user(self, user_id: str) -> User | None: + """Get a user by ID.""" + if user_id in self.active_users: + return self.active_users[user_id] + + # Try to fetch from service + user = self.service.get_user(user_id) + if user: + self.active_users[user.id] = user + return user + + def update_user_stats(self, user_id: str, login_count: int, days_since_active: int) -> None: + """Update statistics for a user.""" + if user_id not in self.user_stats: + self.user_stats[user_id] = UserStats(user_id=user_id) + + stats = self.user_stats[user_id] + stats.login_count = login_count + stats.last_active_days = days_since_active + + # Calculate engagement score based on activity + engagement = (100 - min(days_since_active, 100)) * 0.8 + engagement += min(login_count, 20) * 0.2 + stats.engagement_score = engagement + + def get_active_users(self) -> list[User]: + """Get all active users.""" + active_user_ids = [user_id for user_id, stats in self.user_stats.items() if stats.is_active()] + return [self.active_users[user_id] for user_id in active_user_ids if user_id in self.active_users] + + def get_user_by_email(self, email: str) -> User | None: + """Find a user by their email address.""" + for user in self.active_users.values(): + if user.email == email: + return user + return None + + +# Example function demonstrating type annotations +def process_user_data(users: list[User], include_inactive: bool = False, transform_func: callable | None = None) -> dict[str, Any]: + """Process user data with optional transformations.""" + result: dict[str, Any] = {"users": [], "total": 0, "admin_count": 0} + + for user in users: + if transform_func: + user_data = transform_func(user.to_dict()) + else: + user_data = user.to_dict() + + result["users"].append(user_data) + result["total"] += 1 + + if "admin" in user.roles: + result["admin_count"] += 1 + + return result + + +def main(): + """Main function demonstrating the usage of UserManager.""" + # Initialize service and manager + service = UserService() + manager = UserManager(service) + + # Register some users + admin = manager.register_user("Admin User", "admin@example.com", ["admin"]) + user1 = manager.register_user("Regular User", "user@example.com", ["user"]) + user2 = manager.register_user("Another User", "another@example.com", ["user"]) + + # Update some stats + manager.update_user_stats(admin.id, 100, 5) + manager.update_user_stats(user1.id, 50, 10) + manager.update_user_stats(user2.id, 10, 45) # Inactive user + + # Get active users + active_users = manager.get_active_users() + logger.info(f"Active users: {len(active_users)}") + + # Process user data + user_data = process_user_data(active_users, transform_func=lambda u: {**u, "full_name": u.get("name", "")}) + + logger.info(f"Processed {user_data['total']} users, {user_data['admin_count']} admins") + + # Example of calling create_user directly + external_user = create_user_object(id="ext123", name="External User", email="external@example.org", roles=["external"]) + logger.info(f"Created external user: {external_user.name}") + + +if __name__ == "__main__": + main() From 316357ab91936d44ec02f79c64d8056217c4d8e3 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Mon, 7 Apr 2025 16:47:50 +0200 Subject: [PATCH 05/12] Serena: option to use gitignore for ignoring, defaulting to true --- myproject.template.yml | 9 +++++++-- src/serena/agent.py | 17 +++++++++++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/myproject.template.yml b/myproject.template.yml index 907f2311..5aca9627 100644 --- a/myproject.template.yml +++ b/myproject.template.yml @@ -7,8 +7,13 @@ project_root: /path/to/project # * csharp: Requires the presence of a .sln file in the project folder. language: python -# list of directories to ignore: either names (e.g. "temp") or relative paths (e.g. "build/foo") -ignored_dirs: [".git", "temp"] +# whether to use the project's gitignore file to ignore files +ignore_all_files_in_gitignore: true + +# list of additional paths to ignore +# same syntax as gitignore, so you can use * and ** +ignored_paths: [] + # list of tool names to exclude. We recommend not excluding any tools, see the readme for more details. # Below is the complete list of tools for convenience. diff --git a/src/serena/agent.py b/src/serena/agent.py index e84b7c04..c0e8245d 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -47,8 +47,12 @@ class ProjectConfig(ToStringMixin): if project_root is None: project_root = Path(config_dict["project_root"]) self.project_root: str = str(project_root.resolve()) - self.ignored_dirs: list[str] = config_dict.get("ignored_dirs", []) + self.ignored_paths: list[str] = config_dict.get("ignored_paths", []) self.excluded_tools: set[str] = set(config_dict.get("excluded_tools", [])) + self.ignore_all_files_in_gitignore = config_dict.get("ignore_all_files_in_gitignore", True) + + # backwards compatibility + self.ignored_paths.extend(config_dict.get("ignored_dirs", [])) @classmethod def from_yml(cls, yml_path: Path) -> Self: @@ -251,9 +255,14 @@ class SerenaAgent: # instantiate and start the language server assert self.project_config is not None - multilspy_config = MultilspyConfig(code_language=self.project_config.language) + multilspy_config = MultilspyConfig(code_language=self.project_config.language, ignored_paths=self.project_config.ignored_paths) ls_logger = MultilspyLogger() - self.language_server = SyncLanguageServer.create(multilspy_config, ls_logger, self.project_config.project_root) + self.language_server = SyncLanguageServer.create( + multilspy_config, + ls_logger, + self.project_config.project_root, + add_gitignore_content_to_config=self.project_config.ignore_all_files_in_gitignore, + ) self.language_server.start() if not self.language_server.is_running(): raise RuntimeError(f"Failed to start the language server for {self.project_config}") @@ -544,7 +553,7 @@ class ListDirTool(Tool): os.path.join(self.project_root, relative_path), relative_to=self.project_root, recursive=recursive, - ignored_dirs=self.project_config.ignored_dirs, + ignored_dirs=self.project_config.ignored_paths, ) result = json.dumps({"dirs": dirs, "files": files}) return self._limit_length(result, max_answer_chars) From e4dbd800286ee04d15c1d64d0adf3d922b0d9859 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Mon, 7 Apr 2025 16:53:54 +0200 Subject: [PATCH 06/12] Changelog --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3ec240f..e56d6ad1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ Changes prior to the next official version change will appear here. * New tool: FindReferencingCodeSnippets * Adjusted prompt in CreateTextFileTool to prevent writing partial content (see [here](https://www.reddit.com/r/ClaudeAI/comments/1jpavtm/comment/mloek1x/?utm_source=share&utm_medium=web3x&utm_name=web3xcss&utm_term=1&utm_content=share_button)). * FindSymbolTool: allow passing a file for restricting search, not just a directory (Gemini was too dumb to pass directories) + * Native support for gitignore files for configuring files to be ignored by serena. See also + in *Language Servers* section below. * Allow Serena to switch between projects (project activation) * Add central Serena configuration in `serena_config.yml`, which * contains the list of available projects @@ -18,8 +20,11 @@ Changes prior to the next official version change will appear here. * Logging: * Improve error reporting in case of initialization failure: open a new GUI log window showing the error or ensure that the existing log window remains visible for some time -* Language servers: +* Language Servers: * Fix C# language server initialization issue when the project path contains spaces + * Native support for gitignore in overview, document-tree and find_references operations. + This is an **important** addition, since previously things like `venv` and `node_modules` were scanned + and were likely responsible for slowness of tools and even server crashes (presumably due to OOM errors). * Agno: * Fix Agno reloading mechanism causing failures when initializing the sqlite memory database #8 * Fix Serena GUI log window not capturing logs after initialization From 39a831f63bd48ec9ec07627ade326c2195fd602f Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Mon, 7 Apr 2025 17:45:00 +0200 Subject: [PATCH 07/12] Minor, more logging --- src/serena/agent.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index c0e8245d..aaccbb9c 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -83,11 +83,15 @@ class SerenaConfig: if not os.path.exists(config_file): raise FileNotFoundError(f"Serena configuration file not found: {config_file}") with open(config_file, encoding="utf-8") as f: - config_yaml = yaml.safe_load(f) + try: + log.info(f"Loading Serena configuration from {config_file}") + config_yaml = yaml.safe_load(f) + except Exception as e: + raise ValueError(f"Error loading Serena configuration from {config_file}: {e}") from e # read projects self.projects: dict[str, ProjectConfig] = {} - for project_config_path in config_yaml["projects"]: + for project_config_path in config_yaml.get("projects", []): project_config_path = Path(project_config_path) if not project_config_path.is_absolute(): project_config_path = Path(serena_root_path()) / project_config_path @@ -95,8 +99,12 @@ class SerenaConfig: project_config_path = project_config_path / ProjectConfig.SERENA_MANAGED_DIR / ProjectConfig.SERENA_DEFAULT_PROJECT_FILE if not project_config_path.is_file(): raise FileNotFoundError(f"Project file not found: {project_config_path}") + log.info(f"Loading project configuration from {project_config_path}") project_config = ProjectConfig.from_yml(project_config_path) - self.projects[project_config.project_name] = project_config + try: + self.projects[project_config.project_name] = project_config + except Exception as e: + raise ValueError(f"Error loading project configuration from {project_config_path}: {e}") from e self.project_names = list(self.projects.keys()) self.gui_log_window_enabled = config_yaml.get("gui_log_window", True) @@ -184,7 +192,11 @@ class SerenaAgent: if project_file_path is not None: if not os.path.exists(project_file_path): raise FileNotFoundError(f"Project file not found: {project_file_path}") - project_config = ProjectConfig.from_yml(Path(project_file_path)) + log.info(f"Loading project configuration from {project_file_path}") + try: + project_config = ProjectConfig.from_yml(Path(project_file_path)) + except Exception as e: + raise ValueError(f"Error loading project configuration from {project_file_path}: {e}") from e else: match len(self.serena_config.projects): case 0: From f24fb31d3d420a8e1debdcb410acbdc22096d4f2 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Mon, 7 Apr 2025 17:53:59 +0200 Subject: [PATCH 08/12] Disable GUI window by default Enabling the gui leads to crashes on the linux app. Generally, the user should rather consciously enable the GUI than have it pop up by default --- serena_config.template.yml | 2 +- src/serena/agent.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/serena_config.template.yml b/serena_config.template.yml index 13cffd29..99514c85 100644 --- a/serena_config.template.yml +++ b/serena_config.template.yml @@ -28,7 +28,7 @@ projects: # previous instances. This leads to multiple log windows being opened, and only the last # window being updated. Since we can't control how agno or Claude Desktop start Serena, # we have to live with this limitation for now. -gui_log_window: True +gui_log_window: False # minimum log level for the GUI log window (10 = debug, 20 = info, 30 = warning, 40 = error) gui_log_level: 20 diff --git a/src/serena/agent.py b/src/serena/agent.py index aaccbb9c..c4a3eed2 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -107,7 +107,7 @@ class SerenaConfig: raise ValueError(f"Error loading project configuration from {project_config_path}: {e}") from e self.project_names = list(self.projects.keys()) - self.gui_log_window_enabled = config_yaml.get("gui_log_window", True) + self.gui_log_window_enabled = config_yaml.get("gui_log_window", False) self.gui_log_window_level = config_yaml.get("gui_log_level", logging.INFO) self.enable_project_activation = config_yaml.get("enable_project_activation", True) From a87ff5fad5dd0810b7793089cdaec510096023cc Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Mon, 7 Apr 2025 18:04:09 +0200 Subject: [PATCH 09/12] LS: Minor (docstrings, comments, filter out lines in gitignore) --- src/multilspy/language_server.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/multilspy/language_server.py b/src/multilspy/language_server.py index 93392131..73ba5711 100644 --- a/src/multilspy/language_server.py +++ b/src/multilspy/language_server.py @@ -88,6 +88,10 @@ class LanguageServer: # To be overridden and extended by subclasses def should_always_ignore(self, dirname: str) -> bool: + """ + A language-specific condition for directories that should be ignored always. For example, venv + in Python and node_modules in JS/TS should be ignored always. + """ return dirname.startswith('.') @classmethod @@ -232,9 +236,11 @@ class LanguageServer: # Normalize separators (pathspec expects forward slashes) pattern = pattern.replace(os.path.sep, '/') processed_patterns.append(pattern) - # Combine explicitly passed patterns with + # Combine explicitly passed patterns with the content of the .gitignore file if config.gitignore_file_content is not None: - processed_patterns.extend(config.gitignore_file_content.splitlines()) + for line in config.gitignore_file_content.splitlines(): + if not line.startswith('#') and line.strip() != '': + processed_patterns.append(line.strip()) # Create a pathspec matcher from the processed patterns self.ignore_spec = pathspec.PathSpec.from_lines( @@ -845,8 +851,8 @@ class LanguageServer: For each file, a symbol of kind Module (3) will be created. For directories, a symbol of kind Package (4) will be created. All symbols will have a children attribute, thereby representing the tree structure of all symbols in the project that are within the repository. - Will ignore directories starting with '.', '__pycache__', language-specific defaults (self.default_ignored_dirs), - and user-configured directories (self.config.ignored_dirs). + Will ignore directories starting with '.', language-specific defaults + and user-configured directories (e.g. from .gitignore). :param within_relative_path: pass a relative path to only consider symbols within this path. If a file is passed, only the symbols within this file will be considered. @@ -1685,9 +1691,10 @@ class SyncLanguageServer: Will go through all files in the project and build a tree of symbols. Note: this may be slow the first time it is called. For each file, a symbol of kind Module (3) will be created. For directories, a symbol of kind Package (4) will be created. - All symbols will have a children attribute, thereby representing the tree structure of all symbols in the project + All symbols will have a children attribute, thereby representing the tree structure of all symbols in the project that are within the repository. - Will ignore all directories that start with a dot (.) and __pycache__ directories. + Will ignore directories starting with '.', language-specific defaults + and user-configured directories (e.g. from .gitignore). :param within_relative_path: pass a relative path to only consider symbols within this path. If a file is passed, only the symbols within this file will be considered. @@ -1968,7 +1975,8 @@ class SyncLanguageServer: def should_always_ignore(self, dirname: str) -> bool: """ - Whether the given directory should be ignored. + A language-specific condition for directories that should be ignored always. For example, venv + in Python and node_modules in JS/TS should be ignored always. """ return self.language_server.should_always_ignore(dirname) From 0ef0dd8c4b9c020c0cbf8996ddeb4567548cf8e5 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Mon, 7 Apr 2025 18:33:11 +0200 Subject: [PATCH 10/12] Deduplication and minor fix in logging --- src/serena/agent.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index c4a3eed2..3f45abb0 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -56,8 +56,12 @@ class ProjectConfig(ToStringMixin): @classmethod def from_yml(cls, yml_path: Path) -> Self: - with open(yml_path, encoding="utf-8") as f: - config_dict = yaml.safe_load(f) + log.info(f"Loading project configuration from {yml_path}") + try: + with open(yml_path, encoding="utf-8") as f: + config_dict = yaml.safe_load(f) + except Exception as e: + raise ValueError(f"Error loading project configuration from {yml_path}: {e}") from e if yml_path.parent.name == cls.SERENA_MANAGED_DIR: project_root = yml_path.parent.parent project_name = project_root.name @@ -101,10 +105,7 @@ class SerenaConfig: raise FileNotFoundError(f"Project file not found: {project_config_path}") log.info(f"Loading project configuration from {project_config_path}") project_config = ProjectConfig.from_yml(project_config_path) - try: - self.projects[project_config.project_name] = project_config - except Exception as e: - raise ValueError(f"Error loading project configuration from {project_config_path}: {e}") from e + self.projects[project_config.project_name] = project_config self.project_names = list(self.projects.keys()) self.gui_log_window_enabled = config_yaml.get("gui_log_window", False) @@ -193,10 +194,7 @@ class SerenaAgent: if not os.path.exists(project_file_path): raise FileNotFoundError(f"Project file not found: {project_file_path}") log.info(f"Loading project configuration from {project_file_path}") - try: - project_config = ProjectConfig.from_yml(Path(project_file_path)) - except Exception as e: - raise ValueError(f"Error loading project configuration from {project_file_path}: {e}") from e + project_config = ProjectConfig.from_yml(Path(project_file_path)) else: match len(self.serena_config.projects): case 0: From 3ae7a6ec10411e5454fad77235c9e7319dd18c95 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Mon, 7 Apr 2025 21:17:54 +0200 Subject: [PATCH 11/12] Raise errors on outdated config Added explanation in readme and changelog --- .vscode/settings.json | 1 + CHANGELOG.md | 5 ++++- README.md | 8 ++++++++ myproject.template.yml | 3 +++ src/serena/agent.py | 24 ++++++++++++++++++++---- 5 files changed, 36 insertions(+), 5 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 8a290211..3f32d63a 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,6 +3,7 @@ "agno", "asyncio", "genai", + "getpid", "Gopls", "langsrv", "multilspy", diff --git a/CHANGELOG.md b/CHANGELOG.md index e56d6ad1..5f4be5ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,13 +4,16 @@ Changes prior to the next official version change will appear here. # 2025-04-07 +> **Breaking Config Changes**: make sure to set `ignore_all_files_in_gitignore`, remove `ignore_dirs` +> and (optionally) set `ignore_paths` in your project configs. See [updated config template](myproject.template.yml) + * Serena core: * New tool: FindReferencingCodeSnippets * Adjusted prompt in CreateTextFileTool to prevent writing partial content (see [here](https://www.reddit.com/r/ClaudeAI/comments/1jpavtm/comment/mloek1x/?utm_source=share&utm_medium=web3x&utm_name=web3xcss&utm_term=1&utm_content=share_button)). * FindSymbolTool: allow passing a file for restricting search, not just a directory (Gemini was too dumb to pass directories) * Native support for gitignore files for configuring files to be ignored by serena. See also in *Language Servers* section below. - * Allow Serena to switch between projects (project activation) + * **Major Feature**: Allow Serena to switch between projects (project activation) * Add central Serena configuration in `serena_config.yml`, which * contains the list of available projects * allows to configure whether project activation is enabled diff --git a/README.md b/README.md index 8c8770ac..01724720 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,14 @@ implementation. Coming soon: Kotlin and Dart. + +> ⚠️ **Note:** Serena is under active development, we are continuously adding features, improving stability and UX. +> As a result, configuration might change in a breaking way. If you have an invalid configuration, +> the MCP server (or the Serena Agent) may not start properly (investigate the MCP logs in the former case). +> Check the [changelog](CHANGELOG.md) +> and the configuration templates when updating Serena and update your configs accordingly. + + ## Table of Contents diff --git a/myproject.template.yml b/myproject.template.yml index 5aca9627..51191a45 100644 --- a/myproject.template.yml +++ b/myproject.template.yml @@ -8,10 +8,13 @@ project_root: /path/to/project language: python # whether to use the project's gitignore file to ignore files +# Added on 2025-04-07 ignore_all_files_in_gitignore: true # list of additional paths to ignore # same syntax as gitignore, so you can use * and ** +# Was previously called `ignored_dirs`, please update your config if you are using that. +# Added (renamed)on 2025-04-07 ignored_paths: [] diff --git a/src/serena/agent.py b/src/serena/agent.py index 3f45abb0..a8dca604 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -37,6 +37,10 @@ TTool = TypeVar("TTool", bound="Tool") SUCCESS_RESULT = "OK" +class SerenaConfigError(Exception): + pass + + class ProjectConfig(ToStringMixin): SERENA_MANAGED_DIR = ".serena" SERENA_DEFAULT_PROJECT_FILE = "project.yml" @@ -49,10 +53,20 @@ class ProjectConfig(ToStringMixin): self.project_root: str = str(project_root.resolve()) self.ignored_paths: list[str] = config_dict.get("ignored_paths", []) self.excluded_tools: set[str] = set(config_dict.get("excluded_tools", [])) - self.ignore_all_files_in_gitignore = config_dict.get("ignore_all_files_in_gitignore", True) - # backwards compatibility - self.ignored_paths.extend(config_dict.get("ignored_dirs", [])) + if "ignore_all_files_in_gitignore" not in config_dict: + raise SerenaConfigError( + "`ignore_all_files_in_gitignore` key not found in project configuration. Please update your `project.yml` file." + "It is recommended to set this to `True`." + ) + self.ignore_all_files_in_gitignore = config_dict["ignore_all_files_in_gitignore"] + + # Raise errors for deprecated keys + if "ignored_dirs" in config_dict: + raise SerenaConfigError( + "`ignored_dirs` key is deprecated. Please use `ignored_paths` instead." + "Note that you can also set `ignore_all_files_in_gitignore` to `True`, which will be enough for most cases." + ) @classmethod def from_yml(cls, yml_path: Path) -> Self: @@ -95,7 +109,9 @@ class SerenaConfig: # read projects self.projects: dict[str, ProjectConfig] = {} - for project_config_path in config_yaml.get("projects", []): + if "projects" not in config_yaml: + raise SerenaConfigError("`projects` key not found in Serena configuration. Please update your `serena_config.yml` file.") + for project_config_path in config_yaml["projects"]: project_config_path = Path(project_config_path) if not project_config_path.is_absolute(): project_config_path = Path(serena_root_path()) / project_config_path From 4380ac3ae91f68b4f6ef620bf6943e980b0fa885 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Mon, 7 Apr 2025 21:40:19 +0200 Subject: [PATCH 12/12] Default for gui_log_window back to true Added explanation on limitations with linux or other clients --- serena_config.template.yml | 5 ++++- src/serena/agent.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/serena_config.template.yml b/serena_config.template.yml index 99514c85..5aa220b9 100644 --- a/serena_config.template.yml +++ b/serena_config.template.yml @@ -28,7 +28,10 @@ projects: # previous instances. This leads to multiple log windows being opened, and only the last # window being updated. Since we can't control how agno or Claude Desktop start Serena, # we have to live with this limitation for now. -gui_log_window: False +# +# Limitations: doesn't seem to work with the community version of Claude Desktop for Linux +# Might also cause problems with some MCP clients - if you have any issues, try disabling this +gui_log_window: True # minimum log level for the GUI log window (10 = debug, 20 = info, 30 = warning, 40 = error) gui_log_level: 20 diff --git a/src/serena/agent.py b/src/serena/agent.py index a8dca604..6a86c338 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -124,7 +124,7 @@ class SerenaConfig: self.projects[project_config.project_name] = project_config self.project_names = list(self.projects.keys()) - self.gui_log_window_enabled = config_yaml.get("gui_log_window", False) + self.gui_log_window_enabled = config_yaml.get("gui_log_window", True) self.gui_log_window_level = config_yaml.get("gui_log_level", logging.INFO) self.enable_project_activation = config_yaml.get("enable_project_activation", True)