Merge pull request #510 from oraios/unify-ignore-logic

Improve handling of ignored files and .gitignore discovery

src/serena/project.py

@@ -28,9 +28,7 @@ class Project:
             log.info(f"Using {len(ignored_patterns)} ignored paths from the explicit project configuration.")
             log.debug(f"Ignored paths: {ignored_patterns}")
         if project_config.ignore_all_files_in_gitignore:
-            log.info(f"Parsing all gitignore files in {self.project_root}")
             gitignore_parser = GitignoreParser(self.project_root)
-            log.info(f"Found {len(gitignore_parser.get_ignore_specs())} gitignore files.")
             for spec in gitignore_parser.get_ignore_specs():
                 log.debug(f"Adding {len(spec.patterns)} patterns from {spec.file_path} to the ignored paths.")
                 ignored_patterns.extend(spec.patterns)
@@ -84,9 +82,6 @@ class Project:
         """
         return self._ignore_spec
 
-    def _is_ignored_dirname(self, dirname: str) -> bool:
-        return dirname.startswith(".")
-
     def _is_ignored_relative_path(self, relative_path: str | Path, ignore_non_source_files: bool = True) -> bool:
         """
         Determine whether an existing path should be ignored based on file type and ignore patterns.
@@ -116,16 +111,6 @@ class Project:
         if len(rel_path.parts) > 0 and rel_path.parts[0] == ".git":
             return True
 
-        # Check each part of the path against always fulfilled ignore conditions
-        dir_parts = rel_path.parts
-        if is_file:
-            dir_parts = dir_parts[:-1]
-        for part in dir_parts:
-            if not part:  # Skip empty parts (e.g., from leading '/')
-                continue
-            if self._is_ignored_dirname(part):
-                return True
-
         return match_path(str(relative_path), self.get_ignore_spec(), root_path=self.project_root)
 
     def is_ignored_path(self, path: str | Path, ignore_non_source_files: bool = False) -> bool:
@@ -138,7 +123,13 @@ class Project:
         """
         path = Path(path)
         if path.is_absolute():
-            relative_path = path.relative_to(self.project_root)
+            try:
+                relative_path = path.relative_to(self.project_root)
+            except ValueError:
+                # If the path is not relative to the project root, we consider it as an absolute path outside the project
+                # (which we ignore)
+                log.warning(f"Path {path} is not relative to the project root {self.project_root} and was therefore ignored")
+                return True
         else:
             relative_path = path
 
@@ -193,15 +184,19 @@ class Project:
             return [relative_path]
         else:
             for root, dirs, files in os.walk(start_path, followlinks=True):
-                dirs[:] = [d for d in dirs if not self._is_ignored_relative_path(os.path.join(root, d))]
+                # prevent recursion into ignored directories
+                dirs[:] = [d for d in dirs if not self.is_ignored_path(os.path.join(root, d))]
+
+                # collect non-ignored files
                 for file in files:
-                    rel_file_path = os.path.relpath(os.path.join(root, file), start=self.project_root)
+                    abs_file_path = os.path.join(root, file)
                     try:
-                        if not self._is_ignored_relative_path(rel_file_path):
+                        if not self.is_ignored_path(abs_file_path):
+                            rel_file_path = os.path.relpath(abs_file_path, start=self.project_root)
                             rel_file_paths.append(rel_file_path)
                     except FileNotFoundError:
                         log.warning(
-                            f"File {rel_file_path} not found (possibly due it being a symlink), skipping it in request_parsed_files",
+                            f"File {abs_file_path} not found (possibly due it being a symlink), skipping it in request_parsed_files",
                         )
             return rel_file_paths
 

src/serena/util/file_system.py

@@ -1,7 +1,6 @@
-import glob
 import logging
 import os
-from collections.abc import Callable
+from collections.abc import Callable, Iterator
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import NamedTuple
@@ -138,21 +137,35 @@ class GitignoreParser:
 
     def _load_gitignore_files(self) -> None:
         """Load all gitignore files from the repository."""
-        gitignore_files = self._find_gitignore_files()
-
-        for gitignore_file in gitignore_files:
-            spec = self._create_ignore_spec(gitignore_file)
+        for gitignore_path in self._iter_gitignore_files():
+            log.info("Processing gitignore file: %s", gitignore_path)
+            spec = self._create_ignore_spec(gitignore_path)
             if spec.patterns:  # Only add non-empty specs
                 self.ignore_specs.append(spec)
 
-    def _find_gitignore_files(self) -> list[str]:
+    def _iter_gitignore_files(self, follow_symlinks: bool = False) -> Iterator[str]:
         """
-        Find all .gitignore files in the repository.
+        Iteratively discover .gitignore files in a top-down fashion, starting from the repository root.
+        Directory paths are skipped if they match any already loaded ignore patterns.
 
-        :return: List of absolute paths to .gitignore files
+        :return: an iterator yielding paths to .gitignore files (top-down)
         """
-        relative_paths = glob.glob("**/.gitignore", root_dir=self.repo_root, recursive=True)
-        return [os.path.join(self.repo_root, rel_path) for rel_path in relative_paths]
+        queue: list[str] = [self.repo_root]
+
+        def scan(abs_path: str | None):
+            for entry in os.scandir(abs_path):
+                if entry.is_dir(follow_symlinks=follow_symlinks):
+                    queue.append(entry.path)
+                elif entry.is_file(follow_symlinks=follow_symlinks) and entry.name == ".gitignore":
+                    yield entry.path
+
+        while queue:
+            next_abs_path = queue.pop(0)
+            if next_abs_path != self.repo_root:
+                rel_path = os.path.relpath(next_abs_path, self.repo_root)
+                if self.should_ignore(rel_path):
+                    continue
+            yield from scan(next_abs_path)
 
     def _create_ignore_spec(self, gitignore_file_path: str) -> GitignoreSpec:
         """