feat: ingestion optimisations

application/core/settings.py

@@ -49,6 +49,9 @@ class Settings(BaseSettings):
     EMBEDDINGS_BASE_URL: Optional[str] = None  # Remote embeddings API URL (OpenAI-compatible)
     EMBEDDINGS_KEY: Optional[str] = None  # api key for embeddings (if using openai, just copy API_KEY)
     EMBEDDINGS_MAX_INPUT_TOKENS: Optional[int] = None  # truncate each remote embed input to N tokens (overflow lost)
+    EMBEDDINGS_BATCH_SIZE: int = 32  # chunks per embed request during ingest (1 = legacy per-chunk behaviour)
+    GITHUB_INGEST_MAX_FILE_BYTES: int = 1048576  # skip repo blobs larger than this (0 = no cap)
+    GITHUB_INGEST_MAX_WORKERS: int = 8  # parallel file fetches per GitHub repo ingest
     # Optional directory of operator-supplied model YAMLs, loaded after the
     # built-in catalog under application/core/models/. Later wins on
     # duplicate model id. See application/core/models/README.md.

application/parser/embedding_pipeline.py

@@ -39,31 +39,91 @@ def sanitize_content(content: str) -> str:
     return content.replace('\x00', '')
 
 
-# Per-chunk inline retry. Aggressive defaults (tries=10, delay=60) blocked
+# Fallback when ``settings.EMBEDDINGS_BATCH_SIZE`` is unset or unusable.
+DEFAULT_EMBEDDINGS_BATCH_SIZE = 32
+
+
+def _resolve_batch_size() -> int:
+    """Read ``EMBEDDINGS_BATCH_SIZE``, falling back to the default.
+
+    Tolerates a missing, ``None`` or non-numeric setting (tests patch
+    ``settings`` with a ``MagicMock``) so a misconfiguration degrades to
+    the default batch rather than breaking ingest.
+
+    Returns:
+        Chunks per embed request, always >= 1.
+    """
+    raw = getattr(settings, "EMBEDDINGS_BATCH_SIZE", None)
+    # Explicit type check rather than a bare ``int(raw)``: ``int(MagicMock())``
+    # succeeds and yields 1, which would silently drop ingest back to the
+    # per-chunk behaviour this batching replaces.
+    if isinstance(raw, bool) or not isinstance(raw, (int, str)):
+        return DEFAULT_EMBEDDINGS_BATCH_SIZE
+    try:
+        size = int(raw)
+    except (TypeError, ValueError):
+        return DEFAULT_EMBEDDINGS_BATCH_SIZE
+    return max(1, size)
+
+
+# Per-batch inline retry. Aggressive defaults (tries=10, delay=60) blocked
 # the loop for up to 9 min per chunk and wedged the heartbeat: lower the
 # tail so a transient failure fails-fast and the chunk-progress checkpoint
 # resumes cleanly on next dispatch.
 @retry(tries=3, delay=5, backoff=2)
+def add_texts_to_store_with_retry(
+    store: Any, docs: List[Any], source_id: str
+) -> None:
+    """Add a batch of documents to the vector store with retry logic.
+
+    One call per batch replaces one call per chunk: the remote embeddings
+    API accepts a list, and the store writes the whole batch in a single
+    transaction, so this collapses N HTTP round-trips and N INSERTs into
+    one of each. Safe to retry — ``add_texts`` embeds before it inserts
+    and rolls back on failure, so a failed batch writes nothing.
+
+    Args:
+        store: The vector store object.
+        docs: The documents to be added.
+        source_id: Unique identifier for the source.
+
+    Raises:
+        Exception: If the batch fails after all retry attempts.
+    """
+    if not docs:
+        return
+    try:
+        texts: List[str] = []
+        metadatas: List[Any] = []
+        for doc in docs:
+            # Sanitize content to remove NUL characters that cause ingestion failures
+            doc.page_content = sanitize_content(doc.page_content)
+            doc.metadata["source_id"] = str(source_id)
+            texts.append(doc.page_content)
+            metadatas.append(doc.metadata)
+        store.add_texts(texts, metadatas=metadatas)
+    except Exception as e:
+        logging.error(
+            f"Failed to add {len(docs)} document(s) with retry: {e}", exc_info=True
+        )
+        raise
+
+
 def add_text_to_store_with_retry(store: Any, doc: Any, source_id: str) -> None:
-    """Add a document's text and metadata to the vector store with retry logic.
-    
+    """Add a single document to the vector store with retry logic.
+
+    Thin wrapper over :func:`add_texts_to_store_with_retry`, kept for the
+    per-chunk fallback in the embed loop and for callers outside it.
+
     Args:
         store: The vector store object.
         doc: The document to be added.
         source_id: Unique identifier for the source.
-        
+
     Raises:
         Exception: If document addition fails after all retry attempts.
     """
-    try:
-        # Sanitize content to remove NUL characters that cause ingestion failures
-        doc.page_content = sanitize_content(doc.page_content)
-        
-        doc.metadata["source_id"] = str(source_id)
-        store.add_texts([doc.page_content], metadatas=[doc.metadata])
-    except Exception as e:
-        logging.error(f"Failed to add document with retry: {e}", exc_info=True)
-        raise
+    add_texts_to_store_with_retry(store, [doc], source_id)
 
 
 def _init_progress_and_resume_index(
@@ -115,6 +175,39 @@ def _record_progress(source_id: str, last_index: int, embedded_chunks: int) -> N
         )
 
 
+def _embed_batch_individually(
+    store: Any,
+    docs: List[Any],
+    batch_start: int,
+    batch_end: int,
+    source_id: str,
+) -> tuple[Exception | None, int | None]:
+    """Re-run one failed batch a chunk at a time, checkpointing as it goes.
+
+    Called only after a batch raised. Preserves the pre-batching failure
+    contract: chunks before the offender are embedded and recorded, and the
+    returned index is the exact chunk that failed rather than the batch head.
+
+    Args:
+        store: The vector store object.
+        docs: The full chunk list.
+        batch_start: Index of the first chunk in the failed batch.
+        batch_end: Index one past the last chunk in the failed batch.
+        source_id: Unique identifier for the source.
+
+    Returns:
+        ``(None, None)`` when every chunk succeeded on its own, else the
+        exception and the index of the chunk that failed.
+    """
+    for idx in range(batch_start, batch_end):
+        try:
+            add_text_to_store_with_retry(store, docs[idx], source_id)
+            _record_progress(source_id, last_index=idx, embedded_chunks=idx + 1)
+        except Exception as e:
+            return e, idx
+    return None, None
+
+
 def assert_index_complete(source_id: str) -> None:
     """Raise ``EmbeddingPipelineError`` if ``ingest_chunk_progress``
     shows a partial embed for ``source_id``.
@@ -275,24 +368,31 @@ def embed_and_store_documents(
         # tripwire still validates ``embedded == total`` afterwards.
         loop_start = total_docs
 
-    # Process and embed documents
+    # Process and embed documents, one batch per embed request. Progress is
+    # checkpointed per batch rather than per chunk, so a 3k-chunk ingest
+    # writes ~90 progress rows instead of 3k — the per-chunk bookkeeping
+    # (Neon UPDATE + Celery update_state + SSE) dominated wall-clock more
+    # than the embedding itself.
     chunk_error: Exception | None = None
     failed_idx: int | None = None
     last_published_pct = -1
     source_id_str = str(source_id)
     progress_span = progress_end - progress_start
-    for idx in tqdm(
-        range(loop_start, total_docs),
+    batch_size = _resolve_batch_size()
+    batch_starts = list(range(loop_start, total_docs, batch_size))
+    for batch_start in tqdm(
+        batch_starts,
         desc="Embedding 🦖",
-        unit="docs",
-        total=total_docs - loop_start,
+        unit="batch",
+        total=len(batch_starts),
         bar_format="{l_bar}{bar}| Time Left: {remaining}",
     ):
-        doc = docs[idx]
+        batch_end = min(batch_start + batch_size, total_docs)
+        last_idx = batch_end - 1
         try:
             # Map the embed loop into [progress_start, progress_end].
             progress = progress_start + int(
-                ((idx + 1) / total_docs) * progress_span
+                (batch_end / total_docs) * progress_span
             )
             task_status.update_state(state="PROGRESS", meta={"current": progress})
 
@@ -307,21 +407,45 @@ def embed_and_store_documents(
                     {
                         "current": progress,
                         "total": total_docs,
-                        "embedded_chunks": idx + 1,
+                        "embedded_chunks": batch_end,
                         "stage": "embedding",
                     },
                     scope={"kind": "source", "id": source_id_str},
                 )
                 last_published_pct = progress
 
-            # Add document to vector store
-            add_text_to_store_with_retry(store, doc, source_id)
-            _record_progress(source_id, last_index=idx, embedded_chunks=idx + 1)
-        except Exception as e:
-            chunk_error = e
-            failed_idx = idx
-            logging.error(f"Error embedding document {idx}: {e}", exc_info=True)
-            logging.info(f"Saving progress at document {idx} out of {total_docs}")
+            # Add the batch to the vector store
+            add_texts_to_store_with_retry(
+                store, docs[batch_start:batch_end], source_id
+            )
+            _record_progress(
+                source_id, last_index=last_idx, embedded_chunks=batch_end
+            )
+        except Exception as batch_exc:
+            # The batch failed as a unit and wrote nothing (``add_texts``
+            # embeds before it inserts and rolls back). Re-run it one chunk
+            # at a time so a single poison chunk — an oversized input the
+            # embeddings server rejects — costs only itself: the chunks
+            # before it still land and are checkpointed, and ``failed_idx``
+            # names the real offender instead of the batch head.
+            logging.warning(
+                f"Batch embed failed for chunks {batch_start}-{last_idx} "
+                f"({batch_exc}); retrying individually to isolate the failure"
+            )
+            chunk_error, failed_idx = _embed_batch_individually(
+                store, docs, batch_start, batch_end, source_id
+            )
+            if chunk_error is None:
+                # Every chunk passed on its own — the batch-level failure was
+                # transient (or a payload-size limit). Progress is recorded.
+                continue
+            logging.error(
+                f"Error embedding document {failed_idx}: {chunk_error}",
+                exc_info=True,
+            )
+            logging.info(
+                f"Saving progress at document {failed_idx} out of {total_docs}"
+            )
             try:
                 store.save_local(folder_name)
                 logging.info("Progress saved successfully")

application/parser/remote/github_loader.py

@@ -1,13 +1,53 @@
 import base64
-import requests
+import logging
+import mimetypes
 import time
-from typing import List, Optional
+from concurrent.futures import ThreadPoolExecutor
+from typing import Dict, List, Optional, Tuple
+
+import requests
+
+from application.core.settings import settings
 from application.parser.remote.base import BaseRemote
 from application.parser.schema.base import Document
-import mimetypes
-from application.core.settings import settings
+
+logger = logging.getLogger(__name__)
+
+# Directory names that hold vendored or generated output. Anything under one
+# of these is build product, not source: it bloats the index, costs an API
+# call each, and answers no question a user would ask of the repo.
+SKIP_DIRECTORIES = {
+    ".git", ".github/workflows/generated", ".idea", ".vscode",
+    "__pycache__", ".mypy_cache", ".pytest_cache", ".tox", ".venv", "venv",
+    "node_modules", "bower_components", "vendor", "third_party", "external",
+    "dist", "build", "out", "target", "bin", "obj",
+    "site-packages", "coverage", "htmlcov", ".next", ".nuxt", ".svelte-kit",
+}
+
+# Exact filenames that are machine-generated and semantically empty.
+SKIP_FILENAMES = {
+    "package-lock.json", "yarn.lock", "pnpm-lock.yaml", "bun.lockb",
+    "composer.lock", "gemfile.lock", "cargo.lock", "poetry.lock",
+    "pipfile.lock", "go.sum", "mix.lock", "podfile.lock",
+}
+
+# Suffixes that mark minified or map output even under a source directory.
+SKIP_SUFFIXES = (
+    ".min.js", ".min.css", ".map", ".lock",
+    ".pyc", ".pyo", ".class", ".jar", ".war", ".o", ".so", ".dylib", ".dll",
+    ".exe", ".wasm", ".bin", ".pdb",
+)
+
 
 class GitHubLoader(BaseRemote):
+    """Load a GitHub repository's text files as ``Document`` objects.
+
+    Uses the git *tree* API to enumerate the repository in a single request
+    rather than walking ``/contents/`` directory by directory, filters out
+    binaries, vendored output and oversized blobs *before* fetching them, and
+    downloads the survivors in parallel.
+    """
+
     def __init__(self):
         self.access_token = settings.GITHUB_ACCESS_TOKEN
         self.headers = {
@@ -22,15 +62,22 @@ class GitHubLoader(BaseRemote):
         """Determine if a file is a text file based on extension."""
         # Common text file extensions
         text_extensions = {
-            '.txt', '.md', '.markdown', '.rst', '.json', '.xml', '.yaml', '.yml',
-            '.py', '.js', '.ts', '.jsx', '.tsx', '.java', '.c', '.cpp', '.h', '.hpp',
-            '.cs', '.go', '.rs', '.rb', '.php', '.swift', '.kt', '.scala',
-            '.html', '.css', '.scss', '.sass', '.less',
-            '.sh', '.bash', '.zsh', '.fish',
-            '.sql', '.r', '.m', '.mat',
-            '.ini', '.cfg', '.conf', '.config', '.env',
-            '.gitignore', '.dockerignore', '.editorconfig',
-            '.log', '.csv', '.tsv'
+            '.txt', '.md', '.markdown', '.rst', '.adoc', '.org',
+            '.json', '.jsonc', '.xml', '.yaml', '.yml', '.toml',
+            '.py', '.pyi', '.js', '.mjs', '.cjs', '.ts', '.tsx', '.jsx',
+            '.vue', '.svelte', '.astro',
+            '.java', '.c', '.cc', '.cpp', '.h', '.hpp', '.hxx',
+            '.cs', '.go', '.rs', '.rb', '.php', '.swift', '.kt', '.kts',
+            '.scala', '.sc', '.clj', '.cljs', '.edn', '.ex', '.exs', '.erl',
+            '.hs', '.ml', '.mli', '.fs', '.fsx', '.dart', '.lua', '.jl',
+            '.zig', '.nim', '.v', '.pl', '.pm', '.groovy', '.gradle',
+            '.html', '.htm', '.css', '.scss', '.sass', '.less',
+            '.sh', '.bash', '.zsh', '.fish', '.ps1', '.bat',
+            '.sql', '.graphql', '.gql', '.proto', '.thrift',
+            '.tf', '.tfvars', '.hcl', '.dockerfile', '.cmake', '.mk',
+            '.ini', '.cfg', '.conf', '.config', '.properties',
+            '.gitignore', '.dockerignore', '.editorconfig', '.gitattributes',
+            '.csv', '.tsv',
         }
 
         # Get file extension
@@ -39,6 +86,14 @@ class GitHubLoader(BaseRemote):
             if file_lower.endswith(ext):
                 return True
 
+        # Extension-less files that are conventionally text.
+        basename = file_lower.rsplit("/", 1)[-1]
+        if basename in {
+            "dockerfile", "makefile", "rakefile", "gemfile", "procfile",
+            "license", "licence", "notice", "authors", "codeowners", "readme",
+        }:
+            return True
+
         # Also check MIME type
         mime_type, _ = mimetypes.guess_type(file_path)
         if mime_type and (mime_type.startswith("text") or mime_type in ["application/json", "application/xml"]):
@@ -46,6 +101,179 @@ class GitHubLoader(BaseRemote):
 
         return False
 
+    def should_skip_path(self, file_path: str) -> bool:
+        """Return ``True`` for vendored, generated or binary-by-name paths.
+
+        Applied to the tree listing before any content is fetched, so a
+        skipped file costs nothing. Complements :meth:`is_text_file`, which
+        only knows about extensions.
+
+        Args:
+            file_path: Repo-relative path, e.g. ``"web/dist/app.min.js"``.
+
+        Returns:
+            ``True`` when the path should not be ingested.
+        """
+        lowered = file_path.lower()
+        parts = lowered.split("/")
+        if any(part in SKIP_DIRECTORIES for part in parts[:-1]):
+            return True
+        if parts[-1] in SKIP_FILENAMES:
+            return True
+        if lowered.endswith(SKIP_SUFFIXES):
+            return True
+        # Dot-directories (.git, .cache, ...) never carry documentation.
+        if any(part.startswith(".") and part not in {".github"} for part in parts[:-1]):
+            return True
+        return False
+
+    def _max_file_bytes(self) -> int:
+        """Resolve the per-blob size cap; ``0`` disables it."""
+        raw = getattr(settings, "GITHUB_INGEST_MAX_FILE_BYTES", None)
+        if isinstance(raw, bool) or not isinstance(raw, (int, str)):
+            return 1048576
+        try:
+            return max(0, int(raw))
+        except (TypeError, ValueError):
+            return 1048576
+
+    def _max_workers(self) -> int:
+        """Resolve the parallel-fetch width, clamped to a sane range."""
+        raw = getattr(settings, "GITHUB_INGEST_MAX_WORKERS", None)
+        if isinstance(raw, bool) or not isinstance(raw, (int, str)):
+            return 8
+        try:
+            return max(1, min(32, int(raw)))
+        except (TypeError, ValueError):
+            return 8
+
+    @staticmethod
+    def normalize_repo(repo_url: str) -> str:
+        """Reduce a user-pasted repo URL to ``owner/name``.
+
+        Strips the scheme/host, a trailing ``.git`` and any trailing slash —
+        the three shapes that previously 404'd the contents API.
+
+        Args:
+            repo_url: Anything from ``owner/name`` to
+                ``https://github.com/owner/name.git/``.
+
+        Returns:
+            The ``owner/name`` segment.
+        """
+        repo = (repo_url or "").strip()
+        if not repo:
+            return ""
+
+        if repo.startswith("git@"):
+            # git@github.com:owner/name.git
+            host, _, path = repo[len("git@"):].partition(":")
+            if host.lower() != "github.com":
+                return ""
+            repo = path
+        elif "://" in repo:
+            remainder = repo.split("://", 1)[1]
+            host, _, path = remainder.partition("/")
+            # Anything not hosted on github.com is not a repo. Previously such
+            # a URL was pasted straight into the contents path and 404-looped.
+            if host.lower() not in {"github.com", "www.github.com"}:
+                return ""
+            repo = path
+        elif repo.lower().startswith(("github.com/", "www.github.com/")):
+            repo = repo.split("/", 1)[1]
+
+        repo = repo.strip("/")
+        if repo.lower().endswith(".git"):
+            repo = repo[: -len(".git")]
+        parts = [p for p in repo.split("/") if p]
+        if len(parts) < 2:
+            return ""
+        owner, name = parts[0], parts[1]
+        if name.lower().endswith(".git"):
+            name = name[: -len(".git")]
+        # A bare host slipping through ("suat-handbook.netlify.app") has no
+        # owner segment, so require both halves to look like path segments.
+        if not owner or not name or ":" in owner or ":" in name:
+            return ""
+        return f"{owner}/{name}"
+
+    def get_default_branch(self, repo_name: str) -> str:
+        """Return the repo's default branch, falling back to ``main``.
+
+        The blob URL used for citations was hard-coded to ``main``, which
+        produced dead source links for every ``master``-default repo.
+        """
+        try:
+            response = self._make_request(f"https://api.github.com/repos/{repo_name}")
+            branch = response.json().get("default_branch")
+            if branch:
+                return str(branch)
+        except Exception as e:
+            logger.warning(
+                "Could not resolve default branch for %s (%s); assuming 'main'",
+                repo_name, e,
+            )
+        return "main"
+
+    def fetch_repo_tree(
+        self, repo_name: str, branch: str
+    ) -> Tuple[List[Tuple[str, int]], bool]:
+        """List every blob in the repo with one recursive tree request.
+
+        Replaces the per-directory ``/contents/`` walk, which cost one API
+        call per directory (720 for a mid-size repo) before a single file
+        was read.
+
+        Args:
+            repo_name: ``owner/name``.
+            branch: Branch or ref to enumerate.
+
+        Returns:
+            ``(entries, truncated)`` where ``entries`` is a list of
+            ``(path, size_bytes)`` and ``truncated`` flags a repo too large
+            for the tree endpoint (caller should fall back to the walk).
+        """
+        url = (
+            f"https://api.github.com/repos/{repo_name}/git/trees/"
+            f"{branch}?recursive=1"
+        )
+        response = self._make_request(url)
+        payload = response.json()
+        if isinstance(payload, dict) and "tree" not in payload and "message" in payload:
+            raise Exception(f"GitHub API error: {payload.get('message')}")
+        entries = [
+            (item.get("path", ""), int(item.get("size") or 0))
+            for item in payload.get("tree", [])
+            if item.get("type") == "blob"
+        ]
+        return entries, bool(payload.get("truncated"))
+
+    def select_files(self, entries: List[Tuple[str, int]]) -> List[str]:
+        """Filter tree entries down to the paths worth fetching.
+
+        Args:
+            entries: ``(path, size_bytes)`` pairs from :meth:`fetch_repo_tree`.
+
+        Returns:
+            Repo-relative paths that pass the skip-list, the text-extension
+            check and the size cap, in tree order.
+        """
+        max_bytes = self._max_file_bytes()
+        selected: List[str] = []
+        skipped_big = 0
+        for path, size in entries:
+            if not path or self.should_skip_path(path) or not self.is_text_file(path):
+                continue
+            if max_bytes and size > max_bytes:
+                skipped_big += 1
+                continue
+            selected.append(path)
+        if skipped_big:
+            logger.info(
+                "Skipped %d file(s) over the %d-byte cap", skipped_big, max_bytes
+            )
+        return selected
+
     def fetch_file_content(self, repo_url: str, file_path: str) -> Optional[str]:
         """Fetch file content. Returns None if file should be skipped (binary files or empty files)."""
         url = f"https://api.github.com/repos/{repo_url}/contents/{file_path}"
@@ -91,13 +319,18 @@ class GitHubLoader(BaseRemote):
                     remaining = response.headers.get("X-RateLimit-Remaining", "unknown")
                     reset_time = response.headers.get("X-RateLimit-Reset", "unknown")
 
-                    print(f"GitHub API 403 Error: {error_msg}")
-                    print(f"Rate limit remaining: {remaining}, Reset time: {reset_time}")
+                    logger.warning("GitHub API 403 Error: %s", error_msg)
+                    logger.warning(
+                        "Rate limit remaining: %s, Reset time: %s", remaining, reset_time
+                    )
 
                     if "rate limit" in error_msg.lower():
                         if attempt < max_retries - 1:
                             wait_time = 2 ** attempt  # Exponential backoff
-                            print(f"Rate limit hit, waiting {wait_time} seconds before retry...")
+                            logger.warning(
+                                "Rate limit hit, waiting %s seconds before retry...",
+                                wait_time,
+                            )
                             time.sleep(wait_time)
                             continue
 
@@ -111,12 +344,27 @@ class GitHubLoader(BaseRemote):
                         raise
                     # If we can't parse the response, raise the original error
                     response.raise_for_status()
+            elif response.status_code == 401 and self.access_token:
+                # An expired or revoked PAT makes even public repos 401, which
+                # is strictly worse than not sending one. Retry unauthenticated
+                # so a stale credential degrades instead of failing the ingest.
+                logger.warning(
+                    "GitHub rejected the configured token (401); "
+                    "retrying %s unauthenticated", url,
+                )
+                anon = {"Accept": "application/vnd.github.v3+json"}
+                anon_response = requests.get(url, headers=anon, timeout=100)
+                if anon_response.status_code == 200:
+                    return anon_response
+                anon_response.raise_for_status()
+                return anon_response
             else:
                 response.raise_for_status()
 
         return response
 
     def fetch_repo_files(self, repo_url: str, path: str = "") -> List[str]:
+        """Walk ``/contents/`` recursively (fallback for truncated trees)."""
         url = f"https://api.github.com/repos/{repo_url}/contents/{path}"
         response = self._make_request(url)
 
@@ -138,21 +386,69 @@ class GitHubLoader(BaseRemote):
                 files.extend(self.fetch_repo_files(repo_url, item["path"]))
         return files
 
+    def _list_candidate_files(self, repo_name: str, branch: str) -> List[str]:
+        """Enumerate ingestable paths, preferring the single tree request."""
+        try:
+            entries, truncated = self.fetch_repo_tree(repo_name, branch)
+            if not truncated:
+                return self.select_files(entries)
+            logger.warning(
+                "Tree for %s is truncated; falling back to the directory walk",
+                repo_name,
+            )
+        except Exception as e:
+            logger.warning(
+                "Tree listing failed for %s (%s); falling back to the "
+                "directory walk", repo_name, e,
+            )
+        paths = self.fetch_repo_files(repo_name)
+        return self.select_files([(p, 0) for p in paths])
+
     def load_data(self, repo_url: str) -> List[Document]:
-        repo_name = repo_url.split("github.com/")[-1]
-        files = self.fetch_repo_files(repo_name)
+        """Load every ingestable text file in ``repo_url`` as a Document."""
+        repo_name = self.normalize_repo(repo_url)
+        if not repo_name or "/" not in repo_name:
+            raise ValueError(
+                f"Not a valid GitHub repository: {repo_url!r}. "
+                "Expected a github.com URL like https://github.com/owner/name."
+            )
+        branch = self.get_default_branch(repo_name)
+        files = self._list_candidate_files(repo_name, branch)
+        logger.info(
+            "Fetching %d file(s) from %s@%s", len(files), repo_name, branch
+        )
+
+        # Fetch in parallel: this phase is pure network latency, and it was
+        # previously one blocking round-trip per file.
+        contents: Dict[str, Optional[str]] = {}
+
+        def _fetch(file_path: str) -> Tuple[str, Optional[str]]:
+            try:
+                return file_path, self.fetch_file_content(repo_name, file_path)
+            except Exception as e:
+                # One unreadable file must not sink the whole repo ingest.
+                logger.warning("Skipping %s: %s", file_path, e)
+                return file_path, None
+
+        max_workers = min(self._max_workers(), len(files)) or 1
+        with ThreadPoolExecutor(max_workers=max_workers) as pool:
+            for file_path, content in pool.map(_fetch, files):
+                contents[file_path] = content
+
         documents = []
         for file_path in files:
-            content = self.fetch_file_content(repo_name, file_path)
+            content = contents.get(file_path)
             # Skip binary files (content is None)
-            if content is None:
+            if not content:
                 continue
             documents.append(Document(
                 text=content,
                 doc_id=file_path,
                 extra_info={
                     "title": file_path,
-                    "source": f"https://github.com/{repo_name}/blob/main/{file_path}"
+                    "source": (
+                        f"https://github.com/{repo_name}/blob/{branch}/{file_path}"
+                    ),
                 }
             ))
         return documents

application/worker.py

@@ -75,6 +75,82 @@ RECURSION_DEPTH = 2
 INGEST_HEARTBEAT_INTERVAL_SECONDS = 30
 
 
+def count_structure_files(node: dict) -> int:
+    """Count leaf files in a nested ``directory_structure`` mapping.
+
+    Directories are plain dicts of children; files are dicts carrying a
+    ``token_count`` key. ``len()`` on the root only sees top-level entries,
+    which undercounts any repo with subdirectories.
+
+    Args:
+        node: A ``directory_structure`` mapping (or any subtree of one).
+
+    Returns:
+        Number of file leaves beneath ``node``.
+    """
+    if not isinstance(node, dict):
+        return 0
+    total = 0
+    for value in node.values():
+        if isinstance(value, dict):
+            if "token_count" in value and "size_bytes" in value:
+                total += 1
+            else:
+                total += count_structure_files(value)
+    return total
+
+
+def add_file_to_structure(
+    directory_structure: dict,
+    file_path: str,
+    file_type: str,
+    *,
+    size_bytes: int,
+    token_count: int,
+) -> None:
+    """Insert one chunk's stats into a nested ``directory_structure``.
+
+    Callers feed this *chunks*, so a file larger than one chunk arrives
+    several times. Stats are accumulated rather than overwritten — the
+    previous assignment kept only the final fragment, which made the
+    per-file sizes shown in the UI wrong for every multi-chunk file
+    (a 2.56M-token repo reported 733k).
+
+    Args:
+        directory_structure: Mapping mutated in place.
+        file_path: Repo-relative path, e.g. ``"guides/setup.md"``.
+        file_type: MIME type recorded on first insert.
+        size_bytes: Byte length of this chunk.
+        token_count: Token count of this chunk.
+
+    Returns:
+        None
+    """
+    path_parts = [p for p in file_path.split("/") if p]
+    if not path_parts:
+        return
+    current_level = directory_structure
+    for part in path_parts[:-1]:
+        # Intermediate parts are directories
+        child = current_level.get(part)
+        if not isinstance(child, dict) or "token_count" in child:
+            child = {}
+            current_level[part] = child
+        current_level = child
+
+    leaf = path_parts[-1]
+    existing = current_level.get(leaf)
+    if isinstance(existing, dict) and "token_count" in existing:
+        existing["size_bytes"] += size_bytes
+        existing["token_count"] += token_count
+    else:
+        current_level[leaf] = {
+            "type": file_type,
+            "size_bytes": size_bytes,
+            "token_count": token_count,
+        }
+
+
 def graph_extraction_key(source_id, updated_at) -> str:
     """Build the extract_graph idempotency key for a source's current state.
 
@@ -1264,24 +1340,18 @@ def remote_worker(
 
                 # Build nested directory structure from path
                 # e.g., "guides/setup.md" -> {"guides": {"setup.md": {...}}}
-                path_parts = file_path.split("/")
-                current_level = directory_structure
-                for i, part in enumerate(path_parts):
-                    if i == len(path_parts) - 1:
-                        # Last part is the file
-                        current_level[part] = {
-                            "type": file_type,
-                            "size_bytes": size_bytes,
-                            "token_count": token_count,
-                        }
-                    else:
-                        # Intermediate parts are directories
-                        if part not in current_level:
-                            current_level[part] = {}
-                        current_level = current_level[part]
+                add_file_to_structure(
+                    directory_structure, file_path, file_type,
+                    size_bytes=size_bytes, token_count=token_count,
+                )
 
+        # ``len(directory_structure)`` counts only top-level entries, so a
+        # 1,474-file repo logged "44 files". Count the leaves instead — this
+        # line is the operational signal that a remote ingest succeeded.
         logging.info(
-            f"Built directory structure with {len(directory_structure)} files: "
+            f"Built directory structure with "
+            f"{count_structure_files(directory_structure)} files across "
+            f"{len(directory_structure)} top-level entries: "
             f"{list(directory_structure.keys())}"
         )