Merge remote-tracking branch 'origin/main'

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 Status of the `main` branch. Changes prior to the next official version change will appear here.
 
+Fixes:
+* Ignore `.git` in check for ignored paths and improve performance of `find_all_non_ignored_files`
+
 # 0.1.3
 
 ## Summary

src/serena/cli.py

@@ -12,7 +12,7 @@ from tqdm import tqdm
 
 from serena.agent import SerenaAgent
 from serena.config.context_mode import SerenaAgentContext, SerenaAgentMode
-from serena.config.serena_config import ProjectConfig, SerenaConfig
+from serena.config.serena_config import ProjectConfig, SerenaConfig, SerenaPaths
 from serena.constants import (
     DEFAULT_CONTEXT,
     DEFAULT_MODES,
@@ -142,15 +142,22 @@ class TopLevelCommands(AutoRegisteringGroup):
         # initialize logging, using INFO level initially (will later be adjusted by SerenaAgent according to the config)
         #   * memory log handler (for use by GUI/Dashboard)
         #   * stream handler for stderr (for direct console output, which will also be captured by clients like Claude Desktop)
+        #   * file handler
         # (Note that stdout must never be used for logging, as it is used by the MCP server to communicate with the client.)
         Logger.root.setLevel(logging.INFO)
+        formatter = logging.Formatter(SERENA_LOG_FORMAT)
         memory_log_handler = MemoryLogHandler()
         Logger.root.addHandler(memory_log_handler)
         stderr_handler = logging.StreamHandler(stream=sys.stderr)
-        stderr_handler.formatter = logging.Formatter(SERENA_LOG_FORMAT)
+        stderr_handler.formatter = formatter
         Logger.root.addHandler(stderr_handler)
+        log_path = SerenaPaths().get_next_log_file_path("mcp")
+        file_handler = logging.FileHandler(log_path, mode="w")
+        file_handler.formatter = formatter
+        Logger.root.addHandler(file_handler)
 
         log.info("Initializing Serena MCP server")
+        log.info("Storing logs in %s", log_path)
         project_file = project_file_arg or project
         factory = SerenaMCPFactorySingleProcess(context=context, project=project_file, memory_log_handler=memory_log_handler)
         server = factory.create_mcp_server(

src/serena/config/serena_config.py

@@ -7,6 +7,7 @@ import shutil
 from collections.abc import Iterable
 from copy import deepcopy
 from dataclasses import dataclass, field
+from datetime import datetime
 from functools import cached_property
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Optional, Self, TypeVar
@@ -14,7 +15,7 @@ from typing import TYPE_CHECKING, Any, Optional, Self, TypeVar
 import yaml
 from ruamel.yaml.comments import CommentedMap
 from sensai.util import logging
-from sensai.util.logging import LogTime
+from sensai.util.logging import LogTime, datetime_tag
 from sensai.util.string import ToStringMixin
 
 from serena.constants import (
@@ -30,6 +31,7 @@ from serena.util.inspection import determine_programming_language_composition
 from solidlsp.ls_config import Language
 
 from ..analytics import RegisteredTokenCountEstimator
+from ..util.class_decorators import singleton
 
 if TYPE_CHECKING:
     from ..project import Project
@@ -39,6 +41,30 @@ T = TypeVar("T")
 DEFAULT_TOOL_TIMEOUT: float = 240
 
 
+@singleton
+class SerenaPaths:
+    """
+    Provides paths to various Serena-related directories and files.
+    """
+
+    def __init__(self):
+        self.user_config_dir: str = SERENA_MANAGED_DIR_IN_HOME
+        """
+        the path to the user's Serena configuration directory, which is typically ~/.serena
+        """
+
+    def get_next_log_file_path(self, prefix: str):
+        """
+        :param prefix: the filename prefix indicating the type of the log file
+        :return: the full path to the log file to use
+        """
+        log_dir = os.path.join(self.user_config_dir, "logs", datetime.now().strftime("%Y-%m-%d"))
+        os.makedirs(log_dir, exist_ok=True)
+        return os.path.join(log_dir, prefix + "_" + datetime_tag() + ".txt")
+
+    # TODO: Paths from constants.py should be moved here
+
+
 class ToolSet:
     def __init__(self, tool_names: set[str]) -> None:
         self._tool_names = tool_names

src/serena/constants.py

@@ -8,6 +8,7 @@ _serena_in_home_managed_dir = Path.home() / ".serena"
 
 SERENA_MANAGED_DIR_IN_HOME = str(_serena_in_home_managed_dir)
 
+# TODO: Path-related constants should be moved to SerenaPaths; don't add further constants here.
 REPO_ROOT = str(_repo_root_path)
 PROMPT_TEMPLATES_DIR = str(_serena_pkg_path / "resources" / "config" / "prompt_templates")
 SERENAS_OWN_CONTEXT_YAMLS_DIR = str(_serena_pkg_path / "resources" / "config" / "contexts")
@@ -23,7 +24,6 @@ INTERNAL_MODE_YAMLS_DIR = str(_serena_pkg_path / "resources" / "config" / "inter
 SERENA_DASHBOARD_DIR = str(_serena_pkg_path / "resources" / "dashboard")
 SERENA_ICON_DIR = str(_serena_pkg_path / "resources" / "icons")
 
-
 DEFAULT_ENCODING = "utf-8"
 DEFAULT_CONTEXT = "desktop-app"
 DEFAULT_MODES = ("interactive", "editing")

src/serena/util/file_system.py

@@ -3,6 +3,7 @@ import logging
 import os
 from collections.abc import Callable
 from dataclasses import dataclass, field
+from pathlib import Path
 from typing import NamedTuple
 
 import pathspec
@@ -76,8 +77,10 @@ def find_all_non_ignored_files(repo_root: str) -> list[str]:
     :return: A list of all non-ignored files in the repository
     """
     gitignore_parser = GitignoreParser(repo_root)
-    _, files = scan_directory(repo_root, recursive=True)
-    return [file for file in files if not gitignore_parser.should_ignore(file)]
+    _, files = scan_directory(
+        repo_root, recursive=True, is_ignored_dir=gitignore_parser.should_ignore, is_ignored_file=gitignore_parser.should_ignore
+    )
+    return files
 
 
 @dataclass
@@ -182,10 +185,6 @@ class GitignoreParser:
             if not line or line.lstrip().startswith("#"):
                 continue
 
-            # Handle escaped characters at the beginning
-            if line.startswith(("\\#", "\\!")):
-                line = line[1:]
-
             # Store whether this is a negation pattern
             is_negation = line.startswith("!")
             if is_negation:
@@ -197,11 +196,13 @@ class GitignoreParser:
             if not line:
                 continue
 
-            # Determine if pattern is anchored to the gitignore directory
-            is_anchored = "/" in line[:-1] or line.startswith("/")
+            # Handle escaped characters at the beginning
+            if line.startswith(("\\#", "\\!")):
+                line = line[1:]
 
-            # Remove leading slash for processing
-            if line.startswith("/"):
+            # Determine if pattern is anchored to the gitignore directory and remove leading slash for processing
+            is_anchored = line.startswith("/")
+            if is_anchored:
                 line = line[1:]
 
             # Adjust pattern based on gitignore file location
@@ -256,6 +257,11 @@ class GitignoreParser:
         else:
             rel_path = path
 
+        # Ignore paths inside .git
+        rel_path_first_path = Path(rel_path).parts[0]
+        if rel_path_first_path == ".git":
+            return True
+
         abs_path = os.path.join(self.repo_root, rel_path)
 
         # Normalize path separators

src/serena/util/inspection.py

@@ -19,7 +19,7 @@ def iter_subclasses(cls: type[T], recursive: bool = True) -> Generator[type[T],
             yield from iter_subclasses(subclass, recursive)
 
 
-def determine_programming_language_composition(repo_path: str, rel_path_to_gitignore: str = ".gitignore") -> dict[str, float]:
+def determine_programming_language_composition(repo_path: str) -> dict[str, float]:
     """
     Determine the programming language composition of a repository.
 

src/solidlsp/ls.py

@@ -153,7 +153,7 @@ class SolidLanguageServer(ABC):
 
             ls = CSharpLanguageServer(config, logger, repository_root_path)
         elif config.code_language == Language.CSHARP_OMNISHARP:
-            from solidlsp.language_servers.omnisharp.omnisharp import OmniSharp
+            from solidlsp.language_servers.omnisharp import OmniSharp
 
             ls = OmniSharp(config, logger, repository_root_path)
         elif config.code_language == Language.TYPESCRIPT:
@@ -709,6 +709,50 @@ class SolidLanguageServer(ABC):
             ret.append(ls_types.Location(**new_item))
 
         return ret
+    
+    def request_text_document_diagnostics(
+        self, relative_file_path: str
+    ) -> list[ls_types.Diagnostic]:
+        """
+        Raise a [textDocument/diagnostic](https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_diagnostic) request to the Language Server
+        to find diagnostics for the given file. Wait for the response and return the result.
+
+        :param relative_file_path: The relative path of the file to retrieve diagnostics for
+
+        :return: A list of diagnostics for the file
+        """
+        if not self.server_started:
+            self.logger.log(
+                "request_text_document_diagnostics called before Language Server started",
+                logging.ERROR,
+            )
+            raise LanguageServerException("Language Server not started")
+
+        with self.open_file(relative_file_path):
+            response = self.server.send.text_document_diagnostic(
+                {
+                    LSPConstants.TEXT_DOCUMENT: {
+                        LSPConstants.URI: pathlib.Path(str(PurePath(self.repository_root_path, relative_file_path))).as_uri()
+                    }
+                }
+            )
+
+        if response is None:
+            return []
+
+        assert isinstance(response, dict), f"Unexpected response from Language Server (expected list, got {type(response)}): {response}"
+        ret: list[ls_types.Diagnostic] = []
+        for item in response["items"]:
+            new_item: ls_types.Diagnostic = {
+                "uri": pathlib.Path(str(PurePath(self.repository_root_path, relative_file_path))).as_uri(),
+                "severity": item["severity"],
+                "message": item["message"],
+                "range": item["range"],
+                "code": item["code"]
+            }
+            ret.append(ls_types.Diagnostic(new_item))
+
+        return ret
 
     def retrieve_full_file_content(self, file_path: str) -> str:
         """
@@ -1533,6 +1577,7 @@ class SolidLanguageServer(ABC):
         defining_symbol = self.request_containing_symbol(def_path, def_line, def_col, strict=False, include_body=include_body)
 
         return defining_symbol
+    
 
     @property
     def cache_path(self) -> Path:

src/solidlsp/ls_types.py

@@ -317,3 +317,26 @@ class Hover(TypedDict):
     range: NotRequired[Range]
     """ An optional range inside the text document that is used to
     visualize the hover, e.g. by changing the background color. """
+
+class DiagnosticsSeverity(IntEnum):
+    ERROR = 1
+    WARNING = 2
+    INFORMATION = 3
+    HINT = 4
+
+
+class Diagnostic(TypedDict):
+    """Diagnostic information for a text document."""
+
+    uri: DocumentUri
+    """ The URI of the text document to which the diagnostics apply. """
+    range: Range
+    """ The range of the text document to which the diagnostics apply. """
+    severity: NotRequired[DiagnosticsSeverity]
+    """ The severity of the diagnostic. """
+    message: str
+    """ The diagnostic message. """
+    code: str
+    """ The code of the diagnostic. """
+    source: NotRequired[str]
+    """ The source of the diagnostic, e.g. the name of the tool that produced it. """

src/solidlsp/lsp_protocol_handler/lsp_constants.py

@@ -61,3 +61,9 @@ class LSPConstants:
 
     # key used to represent the location in symbols
     LOCATION = "location"
+
+    # Severity level of the diagnostic
+    SEVERITY = "severity"
+
+    # The message of the diagnostic
+    MESSAGE = "message"

test/serena/util/test_file_system.py

@@ -369,7 +369,7 @@ debug.log
 
         assert "*.log" in patterns
         assert "!important.log" in patterns
-        assert "!/src/keep.log" in patterns
+        assert "!src/keep.log" in patterns
 
     def test_comments_and_empty_lines(self):
         """Test that comments and empty lines are ignored."""
@@ -419,6 +419,38 @@ build/
         assert "#not-a-comment.txt" in patterns
         assert "!not-negation.txt" in patterns
 
+    def test_escaped_negation_patterns(self):
+        test_dir = self.repo_path / "test_escaped_negation"
+        test_dir.mkdir()
+
+        gitignore = test_dir / ".gitignore"
+        gitignore.write_text(
+            """*.log
+\\!not-negation.log
+!actual-negation.log
+"""
+        )
+
+        parser = GitignoreParser(str(test_dir))
+        specs = parser.get_ignore_specs()
+
+        assert len(specs) == 1
+        patterns = specs[0].patterns
+
+        # Key assertions: escaped exclamation becomes literal, real negation preserved
+        assert "!not-negation.log" in patterns  # escaped -> literal
+        assert "!actual-negation.log" in patterns  # real negation preserved
+
+        # Test the actual behavioral difference between escaped and real negation:
+        # *.log pattern should ignore test.log
+        assert parser.should_ignore("test.log")
+
+        # Escaped negation file should still be ignored by *.log pattern
+        assert parser.should_ignore("!not-negation.log")
+
+        # Actual negation should override the *.log pattern
+        assert not parser.should_ignore("actual-negation.log")
+
     def test_glob_patterns(self):
         """Test various glob patterns work correctly."""
         test_dir = self.repo_path / "test_glob"