From ba7943d2ea6ba29de878c1d21bab8878f012d9d4 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 21 Aug 2025 17:45:23 +0200 Subject: [PATCH 1/6] Improve logging --- src/serena/project.py | 2 -- src/serena/util/file_system.py | 5 ++++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/serena/project.py b/src/serena/project.py index cc7b12f6..863a4784 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -28,9 +28,7 @@ class Project: log.info(f"Using {len(ignored_patterns)} ignored paths from the explicit project configuration.") log.debug(f"Ignored paths: {ignored_patterns}") if project_config.ignore_all_files_in_gitignore: - log.info(f"Parsing all gitignore files in {self.project_root}") gitignore_parser = GitignoreParser(self.project_root) - log.info(f"Found {len(gitignore_parser.get_ignore_specs())} gitignore files.") for spec in gitignore_parser.get_ignore_specs(): log.debug(f"Adding {len(spec.patterns)} patterns from {spec.file_path} to the ignored paths.") ignored_patterns.extend(spec.patterns) diff --git a/src/serena/util/file_system.py b/src/serena/util/file_system.py index 1869402c..b82d6f07 100644 --- a/src/serena/util/file_system.py +++ b/src/serena/util/file_system.py @@ -151,8 +151,11 @@ class GitignoreParser: :return: List of absolute paths to .gitignore files """ + log.info("Looking for .gitignore files in %s", self.repo_root) relative_paths = glob.glob("**/.gitignore", root_dir=self.repo_root, recursive=True) - return [os.path.join(self.repo_root, rel_path) for rel_path in relative_paths] + result = [os.path.join(self.repo_root, rel_path) for rel_path in relative_paths] + log.info("Found %s .gitignore files", len(result)) + return result def _create_ignore_spec(self, gitignore_file_path: str) -> GitignoreSpec: """ From 2a388dc22caae874f0d2cbecb3a25a924672f139 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 21 Aug 2025 18:17:28 +0200 Subject: [PATCH 2/6] Incrementally look for .gitignore files in a top-down fashion, skipping directories that are ignored based on files already discovered Fixes #453 Probably fixes #470 --- src/serena/util/file_system.py | 44 ++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/src/serena/util/file_system.py b/src/serena/util/file_system.py index b82d6f07..bc682006 100644 --- a/src/serena/util/file_system.py +++ b/src/serena/util/file_system.py @@ -1,7 +1,6 @@ -import glob import logging import os -from collections.abc import Callable +from collections.abc import Callable, Iterator from dataclasses import dataclass, field from pathlib import Path from typing import NamedTuple @@ -138,24 +137,39 @@ class GitignoreParser: def _load_gitignore_files(self) -> None: """Load all gitignore files from the repository.""" - gitignore_files = self._find_gitignore_files() - - for gitignore_file in gitignore_files: - spec = self._create_ignore_spec(gitignore_file) + for gitignore_path in self._iter_gitignore_files(): + log.info("Processing gitignore file: %s", gitignore_path) + spec = self._create_ignore_spec(gitignore_path) if spec.patterns: # Only add non-empty specs self.ignore_specs.append(spec) - def _find_gitignore_files(self) -> list[str]: - """ - Find all .gitignore files in the repository. + @dataclass + class _QueueItem: + abs_path: str + rel_path: str | None = None - :return: List of absolute paths to .gitignore files + def _iter_gitignore_files(self, follow_symlinks: bool = False) -> Iterator[str]: """ - log.info("Looking for .gitignore files in %s", self.repo_root) - relative_paths = glob.glob("**/.gitignore", root_dir=self.repo_root, recursive=True) - result = [os.path.join(self.repo_root, rel_path) for rel_path in relative_paths] - log.info("Found %s .gitignore files", len(result)) - return result + Iteratively discover .gitignore files in a top-down fashion, starting from the repository root. + Directory paths are skipped if they match any already loaded ignore patterns. + + :return: an iterator yielding paths to .gitignore files (top-down) + """ + queue: list[GitignoreParser._QueueItem] = [self._QueueItem(abs_path=self.repo_root)] + + def scan(abs_path: str, rel_path: str | None): + for entry in os.scandir(abs_path): + if entry.is_dir(follow_symlinks=follow_symlinks): + dir_rel_path = rel_path + "/" + entry.name if rel_path else entry.name + queue.append(self._QueueItem(abs_path=entry.path, rel_path=dir_rel_path)) + elif entry.is_file(follow_symlinks=follow_symlinks) and entry.name == ".gitignore": + yield entry.path + + while queue: + item = queue.pop(0) + if item.rel_path is not None and self.should_ignore(item.rel_path): + continue + yield from scan(item.abs_path, item.rel_path) def _create_ignore_spec(self, gitignore_file_path: str) -> GitignoreSpec: """ From ca8ac45c5e83b41ed5e3cbedb372061401da52d9 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 21 Aug 2025 18:42:43 +0200 Subject: [PATCH 3/6] Remove method _is_ignored_dirname (Randomly ignoring directories starting with '.' is not appropriate) --- src/serena/project.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/serena/project.py b/src/serena/project.py index 863a4784..b2da5c27 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -82,9 +82,6 @@ class Project: """ return self._ignore_spec - def _is_ignored_dirname(self, dirname: str) -> bool: - return dirname.startswith(".") - def _is_ignored_relative_path(self, relative_path: str | Path, ignore_non_source_files: bool = True) -> bool: """ Determine whether an existing path should be ignored based on file type and ignore patterns. @@ -114,16 +111,6 @@ class Project: if len(rel_path.parts) > 0 and rel_path.parts[0] == ".git": return True - # Check each part of the path against always fulfilled ignore conditions - dir_parts = rel_path.parts - if is_file: - dir_parts = dir_parts[:-1] - for part in dir_parts: - if not part: # Skip empty parts (e.g., from leading '/') - continue - if self._is_ignored_dirname(part): - return True - return match_path(str(relative_path), self.get_ignore_spec(), root_path=self.project_root) def is_ignored_path(self, path: str | Path, ignore_non_source_files: bool = False) -> bool: From a603054cb905d8612ef5a7d20156fe8a6ebd9ba7 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 21 Aug 2025 18:52:42 +0200 Subject: [PATCH 4/6] Fix prevention of recursion into ignored directories in gather_source_files (Used _is_ignored_relative_path instead of is_ignored_path) --- src/serena/project.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/serena/project.py b/src/serena/project.py index b2da5c27..d25b47cd 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -178,7 +178,9 @@ class Project: return [relative_path] else: for root, dirs, files in os.walk(start_path, followlinks=True): - dirs[:] = [d for d in dirs if not self._is_ignored_relative_path(os.path.join(root, d))] + # prevent recursion into ignored directories + dirs[:] = [d for d in dirs if not self.is_ignored_path(os.path.join(root, d))] + for file in files: rel_file_path = os.path.relpath(os.path.join(root, file), start=self.project_root) try: From b23d883f4240271e614a772e1158037c4f6b09d9 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 21 Aug 2025 19:00:07 +0200 Subject: [PATCH 5/6] Make sure paths that have invalid relative paths are always ignored Fixes #506 --- src/serena/project.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/serena/project.py b/src/serena/project.py index d25b47cd..f5fea2f5 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -123,7 +123,13 @@ class Project: """ path = Path(path) if path.is_absolute(): - relative_path = path.relative_to(self.project_root) + try: + relative_path = path.relative_to(self.project_root) + except ValueError: + # If the path is not relative to the project root, we consider it as an absolute path outside the project + # (which we ignore) + log.warning(f"Path {path} is not relative to the project root {self.project_root} and was therefore ignored") + return True else: relative_path = path @@ -181,14 +187,16 @@ class Project: # prevent recursion into ignored directories dirs[:] = [d for d in dirs if not self.is_ignored_path(os.path.join(root, d))] + # collect non-ignored files for file in files: - rel_file_path = os.path.relpath(os.path.join(root, file), start=self.project_root) + abs_file_path = os.path.join(root, file) try: - if not self._is_ignored_relative_path(rel_file_path): + if not self.is_ignored_path(abs_file_path): + rel_file_path = os.path.relpath(abs_file_path, start=self.project_root) rel_file_paths.append(rel_file_path) except FileNotFoundError: log.warning( - f"File {rel_file_path} not found (possibly due it being a symlink), skipping it in request_parsed_files", + f"File {abs_file_path} not found (possibly due it being a symlink), skipping it in request_parsed_files", ) return rel_file_paths From 716fbb5296dcd262f73c190cae78f236b47c7c5f Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 21 Aug 2025 22:04:35 +0200 Subject: [PATCH 6/6] Remove queue item representation, using absolute paths instead --- src/serena/util/file_system.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/serena/util/file_system.py b/src/serena/util/file_system.py index bc682006..f855f5ef 100644 --- a/src/serena/util/file_system.py +++ b/src/serena/util/file_system.py @@ -143,11 +143,6 @@ class GitignoreParser: if spec.patterns: # Only add non-empty specs self.ignore_specs.append(spec) - @dataclass - class _QueueItem: - abs_path: str - rel_path: str | None = None - def _iter_gitignore_files(self, follow_symlinks: bool = False) -> Iterator[str]: """ Iteratively discover .gitignore files in a top-down fashion, starting from the repository root. @@ -155,21 +150,22 @@ class GitignoreParser: :return: an iterator yielding paths to .gitignore files (top-down) """ - queue: list[GitignoreParser._QueueItem] = [self._QueueItem(abs_path=self.repo_root)] + queue: list[str] = [self.repo_root] - def scan(abs_path: str, rel_path: str | None): + def scan(abs_path: str | None): for entry in os.scandir(abs_path): if entry.is_dir(follow_symlinks=follow_symlinks): - dir_rel_path = rel_path + "/" + entry.name if rel_path else entry.name - queue.append(self._QueueItem(abs_path=entry.path, rel_path=dir_rel_path)) + queue.append(entry.path) elif entry.is_file(follow_symlinks=follow_symlinks) and entry.name == ".gitignore": yield entry.path while queue: - item = queue.pop(0) - if item.rel_path is not None and self.should_ignore(item.rel_path): - continue - yield from scan(item.abs_path, item.rel_path) + next_abs_path = queue.pop(0) + if next_abs_path != self.repo_root: + rel_path = os.path.relpath(next_abs_path, self.repo_root) + if self.should_ignore(rel_path): + continue + yield from scan(next_abs_path) def _create_ignore_spec(self, gitignore_file_path: str) -> GitignoreSpec: """