refactor: rename the application package to docsgpt

The backend import package is now docsgpt, the name it will carry on PyPI;
application was far too generic to install into anyone's site-packages.
git mv plus a mechanical rewrite of every import, dotted string and path
reference: 734 Python files, the compose files, Dockerfile, workflows, docs,
setup scripts, devcontainer, k8s manifests, vscode config, pytest and coverage
config, .gitignore. Behaviour is unchanged.

Kept for one release:
- A top-level application package whose meta-path finder resolves
  application.x.y to the already-imported docsgpt.x.y object, so old imports
  and entry points (celery -A application.app.celery,
  uvicorn application.asgi:asgi_app) keep working with a FutureWarning.
- Celery registers every application.* task name as an alias of its
  docsgpt.* task on start-up, so messages queued by the previous release still
  run. The redbeat key prefix moves to redbeat:docsgpt:v2: so schedule entries
  the previous release wrote are left unread instead of firing twice.

The backend image builds from the repository root (docker build -f
docsgpt/Dockerfile .) so it can ship the alias package; a root .dockerignore
allow-lists docsgpt/ and application/ and keeps caches, local data, .env
files, the sample index files and the Dockerfile out. Compose and the image
workflows point at the new context.

docsgpt/parser/__init__.py

@@ -0,0 +1 @@
+

docsgpt/parser/chunking.py

@@ -0,0 +1,157 @@
+import re
+from typing import List, Tuple
+import logging
+from docsgpt.parser.chunking_creator import ChunkerCreator
+from docsgpt.parser.schema.base import Document
+from docsgpt.parser.tokenization import get_token_counter
+
+logger = logging.getLogger(__name__)
+
+# Smallest share of ``max_tokens`` a chunk must keep for body text when the
+# header is duplicated onto every chunk. A header that leaves less than this
+# makes each chunk mostly repeated header and multiplies the chunk count -- at
+# a budget of 32 out of 1250 a document splits into 39x more chunks than it
+# needs -- so duplication is dropped rather than honoured.
+_MIN_BODY_BUDGET_RATIO = 0.25
+
+
+class Chunker:
+    """Classic token-window chunker (registered as ``classic_chunk``).
+
+    Strategy dispatch lives in ``ChunkerCreator``; this class is one
+    registered implementation. The ``chunking_strategy`` arg is retained for
+    backward-compatible construction and is not used for dispatch here.
+    """
+
+    def __init__(
+        self,
+        chunking_strategy: str = "classic_chunk",
+        max_tokens: int = 2000,
+        min_tokens: int = 150,
+        duplicate_headers: bool = False,
+    ):
+        self.chunking_strategy = chunking_strategy
+        # A budget below 1 would ask for a chunk per token; the strategy
+        # chunkers clamp the same way.
+        self.max_tokens = max(1, int(max_tokens))
+        self.min_tokens = min_tokens
+        self.duplicate_headers = duplicate_headers
+        # Counted in the embedding model's tokenizer, not cl100k: ``max_tokens``
+        # is compared against a limit the embedding server enforces in its own
+        # units, so counting in any other unit is a guess.
+        self.counter = get_token_counter()
+
+    def separate_header_and_body(self, text: str) -> Tuple[str, str]:
+        header_pattern = r"^(.*?\n){3}"
+        match = re.match(header_pattern, text)
+        if match:
+            header = match.group(0)
+            body = text[len(header):]
+        else:
+            header, body = "", text  # No header, treat entire text as body
+        return header, body
+
+
+
+    def split_document(self, doc: Document) -> List[Document]:
+        """Split one oversized document into ``max_tokens``-sized chunks.
+
+        Pieces are sliced out of the original text rather than decoded back
+        from token ids. WordPiece tokenizers normalise as they decode --
+        all-mpnet-base-v2 lowercases -- so a decode round-trip would rewrite
+        every stored document.
+        """
+        header, body = self.separate_header_and_body(doc.text)
+        header_tokens = self.counter.count(header) if header else 0
+
+        if header and header_tokens >= self.max_tokens:
+            # The header alone fills the budget, so no cut of the body can keep
+            # a chunk within it and duplicating it would leave a one-token body
+            # budget -- a chunk per body token. It is only the first three
+            # lines, not something worth preserving at that cost, so it goes
+            # back to being ordinary text and the document splits evenly.
+            logger.warning(
+                "Header of %s is %d token(s), at or over the %d-token chunk "
+                "budget; treating it as body text.",
+                doc.doc_id,
+                header_tokens,
+                self.max_tokens,
+            )
+            body = f"{header}{body}"
+            header, header_tokens = "", 0
+
+        # A chunk carrying the header has that much less room for body text.
+        with_header_budget = max(1, self.max_tokens - header_tokens)
+        duplicate_headers = self.duplicate_headers
+        if duplicate_headers and with_header_budget < self.max_tokens * _MIN_BODY_BUDGET_RATIO:
+            logger.warning(
+                "Header of %s leaves only %d of %d tokens for body text; "
+                "carrying it on the first chunk only.",
+                doc.doc_id,
+                with_header_budget,
+                self.max_tokens,
+            )
+            duplicate_headers = False
+
+        if duplicate_headers:
+            body_pieces = self.counter.split(body, with_header_budget)
+        else:
+            body_pieces = self.counter.split(
+                body, self.max_tokens, first_max_tokens=with_header_budget
+            )
+
+        if not body_pieces and header:
+            # Nothing but a header: the loop below only ever emits the header
+            # attached to a body piece, so without this the document is dropped
+            # from the index entirely.
+            body_pieces = [""]
+
+        split_docs = []
+        for part_index, piece in enumerate(body_pieces):
+            include_header = bool(header) and (duplicate_headers or part_index == 0)
+            chunk_text = f"{header}{piece}" if include_header else piece
+            split_docs.append(
+                Document(
+                    text=chunk_text,
+                    doc_id=f"{doc.doc_id}-{part_index}",
+                    embedding=doc.embedding,
+                    extra_info={
+                        **(doc.extra_info or {}),
+                        "token_count": self.counter.count(chunk_text),
+                    },
+                )
+            )
+        return split_docs
+
+    def classic_chunk(self, documents: List[Document]) -> List[Document]:
+        processed_docs = []
+        i = 0
+        while i < len(documents):
+            doc = documents[i]
+            token_count = self.counter.count(doc.text)
+
+            if self.min_tokens <= token_count <= self.max_tokens:
+                doc.extra_info = doc.extra_info or {}
+                doc.extra_info["token_count"] = token_count
+                processed_docs.append(doc)
+                i += 1
+            elif token_count < self.min_tokens:
+
+                doc.extra_info = doc.extra_info or {}
+                doc.extra_info["token_count"] = token_count
+                processed_docs.append(doc)
+                i += 1
+            else:
+                # Split large documents
+                processed_docs.extend(self.split_document(doc))
+                i += 1
+        return processed_docs
+
+    def chunk(
+        self,
+        documents: List[Document]
+    ) -> List[Document]:
+        return self.classic_chunk(documents)
+
+
+ChunkerCreator.register("classic_chunk", Chunker)

docsgpt/parser/chunking_creator.py

@@ -0,0 +1,57 @@
+"""String-keyed registry for chunking strategies.
+
+Mirrors ``RetrieverCreator``: features register new strategies (``recursive``,
+``markdown``, ``parent_child``, ...) without touching the dispatch site. The
+classic strategy is registered under ``classic_chunk`` by ``chunking.py``.
+"""
+
+from __future__ import annotations
+
+from typing import Type
+
+
+class ChunkerCreator:
+    chunkers: dict[str, Type] = {}
+    _strategies_loaded: bool = False
+
+    @classmethod
+    def _ensure_builtin(cls) -> None:
+        """Register built-in chunkers if they are not registered yet.
+
+        Self-bootstraps so ``create_chunker`` works regardless of import order:
+        ``docsgpt.parser.chunking`` registers ``classic_chunk`` and
+        ``docsgpt.parser.chunking_strategies`` registers ``recursive`` /
+        ``markdown`` / ``parent_child``.
+        """
+        if not cls.chunkers:
+            import docsgpt.parser.chunking  # noqa: F401  (registers classic_chunk)
+        if not cls._strategies_loaded:
+            cls._strategies_loaded = True
+            import docsgpt.parser.chunking_strategies  # noqa: F401
+
+    @classmethod
+    def create_chunker(cls, strategy: str, *args, **kwargs):
+        """Instantiate the chunker registered under ``strategy``.
+
+        Args:
+            strategy: Registry key (e.g. ``classic_chunk``).
+            *args: Positional args forwarded to the chunker constructor.
+            **kwargs: Keyword args forwarded to the chunker constructor.
+
+        Returns:
+            A chunker instance exposing ``chunk(documents) -> List[Document]``.
+
+        Raises:
+            ValueError: If no chunker is registered for ``strategy``.
+        """
+        cls._ensure_builtin()
+        key = (strategy or "classic_chunk").lower()
+        chunker_class = cls.chunkers.get(key)
+        if not chunker_class:
+            raise ValueError(f"No chunker class found for strategy {strategy}")
+        return chunker_class(*args, **kwargs)
+
+    @classmethod
+    def register(cls, key: str, chunker_class: Type) -> None:
+        """Register ``chunker_class`` under ``key`` (idempotent)."""
+        cls.chunkers[key] = chunker_class

docsgpt/parser/chunking_strategies.py

@@ -0,0 +1,320 @@
+"""Additional chunking strategies registered with ``ChunkerCreator``.
+
+Each strategy honours ``max_tokens`` / ``min_tokens`` and reuses the classic
+``Chunker``'s tiktoken encoding for token counting, so token budgets stay
+consistent across strategies. Selecting a strategy is ingest-time only;
+changing it requires a re-ingest (D8). Registered keys: ``recursive``,
+``markdown``, ``parent_child``, ``semantic``.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import List
+
+from docsgpt.parser.chunking import Chunker
+from docsgpt.parser.chunking_creator import ChunkerCreator
+from docsgpt.parser.schema.base import Document
+from docsgpt.parser.tokenization import get_token_counter
+
+logger = logging.getLogger(__name__)
+
+
+class _BaseStrategyChunker:
+    """Shared token helpers for strategy chunkers.
+
+    Mirrors the classic ``Chunker`` constructor so the worker can build any
+    strategy with the same kwargs. ``chunking_strategy`` is accepted for
+    construction compatibility and not used for dispatch (dispatch lives in
+    ``ChunkerCreator``).
+    """
+
+    def __init__(
+        self,
+        chunking_strategy: str = "classic_chunk",
+        max_tokens: int = 2000,
+        min_tokens: int = 150,
+        duplicate_headers: bool = False,
+    ):
+        self.chunking_strategy = chunking_strategy
+        self.max_tokens = max(1, int(max_tokens))
+        self.min_tokens = max(0, int(min_tokens))
+        self.duplicate_headers = duplicate_headers
+        # Same unit as the embedding server counts in; see
+        # ``docsgpt.parser.tokenization``.
+        self.counter = get_token_counter()
+
+    def _token_count(self, text: str) -> int:
+        return self.counter.count(text)
+
+    def _split_by_tokens(self, text: str) -> List[str]:
+        """Split ``text`` into pieces no larger than ``max_tokens`` tokens."""
+        return self.counter.split(text, self.max_tokens)
+
+    def _emit(self, base: Document, part_index: int, text: str) -> Document:
+        """Build a child Document carrying token_count and inherited info."""
+        return Document(
+            text=text,
+            doc_id=f"{base.doc_id}-{part_index}" if base.doc_id else None,
+            embedding=base.embedding,
+            extra_info={
+                **(base.extra_info or {}),
+                "token_count": self._token_count(text),
+            },
+        )
+
+    def _merge_to_min(self, pieces: List[str], joiner: str) -> List[str]:
+        """Accumulate pieces up to ``max_tokens``, flushing past ``min_tokens``."""
+        merged: List[str] = []
+        buffer = ""
+        for piece in pieces:
+            candidate = f"{buffer}{joiner}{piece}" if buffer else piece
+            if self._token_count(candidate) <= self.max_tokens:
+                buffer = candidate
+            else:
+                if buffer:
+                    merged.append(buffer)
+                buffer = piece
+            if buffer and self._token_count(buffer) >= self.min_tokens:
+                merged.append(buffer)
+                buffer = ""
+        if buffer:
+            merged.append(buffer)
+        return merged
+
+
+class RecursiveChunker(_BaseStrategyChunker):
+    """Split on a separator hierarchy, capping at ``max_tokens``.
+
+    Tries paragraph, line, then sentence boundaries before falling back to a
+    hard token split, and merges adjacent fragments while their combined size
+    stays under ``max_tokens`` so chunks clear ``min_tokens`` where possible.
+    """
+
+    _SEPARATORS = ["\n\n", "\n", ". "]
+
+    def _recursive_split(self, text: str, sep_idx: int) -> List[str]:
+        if self._token_count(text) <= self.max_tokens:
+            return [text] if text.strip() else []
+        if sep_idx >= len(self._SEPARATORS):
+            return [p for p in self._split_by_tokens(text) if p.strip()]
+        sep = self._SEPARATORS[sep_idx]
+        parts = text.split(sep)
+        out: List[str] = []
+        for i, part in enumerate(parts):
+            piece = part + sep if i < len(parts) - 1 else part
+            if not piece.strip():
+                continue
+            if self._token_count(piece) <= self.max_tokens:
+                out.append(piece)
+            else:
+                out.extend(self._recursive_split(piece, sep_idx + 1))
+        return out
+
+    def _merge(self, fragments: List[str]) -> List[str]:
+        """Merge small fragments up to ``max_tokens`` to clear ``min_tokens``."""
+        return self._merge_to_min(fragments, "")
+
+    def chunk(self, documents: List[Document]) -> List[Document]:
+        processed: List[Document] = []
+        for doc in documents:
+            fragments = self._recursive_split(doc.text, 0)
+            for idx, text in enumerate(self._merge(fragments)):
+                processed.append(self._emit(doc, idx, text))
+        return processed
+
+
+class MarkdownChunker(_BaseStrategyChunker):
+    """Split on markdown heading boundaries, then token-cap oversized sections.
+
+    Each ``^#{1,6}\\s`` heading starts a new section; sections over
+    ``max_tokens`` are further split by token window so no chunk exceeds the
+    cap.
+    """
+
+    _HEADING = re.compile(r"^#{1,6}\s", re.MULTILINE)
+
+    def _sections(self, text: str) -> List[str]:
+        boundaries = [m.start() for m in self._HEADING.finditer(text)]
+        if not boundaries:
+            return [text] if text.strip() else []
+        if boundaries[0] != 0:
+            boundaries = [0] + boundaries
+        sections = []
+        for i, start in enumerate(boundaries):
+            end = boundaries[i + 1] if i + 1 < len(boundaries) else len(text)
+            section = text[start:end]
+            if section.strip():
+                sections.append(section)
+        return sections
+
+    def chunk(self, documents: List[Document]) -> List[Document]:
+        processed: List[Document] = []
+        for doc in documents:
+            part_index = 0
+            for section in self._sections(doc.text):
+                if self._token_count(section) <= self.max_tokens:
+                    processed.append(self._emit(doc, part_index, section))
+                    part_index += 1
+                else:
+                    for piece in self._split_by_tokens(section):
+                        if not piece.strip():
+                            continue
+                        processed.append(self._emit(doc, part_index, piece))
+                        part_index += 1
+        return processed
+
+
+class ParentChildChunker(_BaseStrategyChunker):
+    """Emit small child chunks for embedding with a larger parent window.
+
+    The document is first split into parent windows of ``max_tokens`` tokens;
+    each window is then split into children of ``min_tokens`` (a sane floor of
+    50) tokens. Each child stashes its parent window text in
+    ``extra_info["parent_text"]`` so retrieval can expand to the parent later.
+    The child text is what gets embedded; ``parent_text`` rides through
+    ``Document.to_vector_format`` into vector-store metadata.
+    """
+
+    def _child_size(self) -> int:
+        size = self.min_tokens if self.min_tokens > 0 else 50
+        return min(size, self.max_tokens)
+
+    def chunk(self, documents: List[Document]) -> List[Document]:
+        processed: List[Document] = []
+        child_size = self._child_size()
+        for doc in documents:
+            part_index = 0
+            for parent_text in self.counter.split(doc.text, self.max_tokens):
+                if not parent_text.strip():
+                    continue
+                for child_text in self.counter.split(parent_text, child_size):
+                    if not child_text.strip():
+                        continue
+                    child = Document(
+                        text=child_text,
+                        doc_id=(
+                            f"{doc.doc_id}-{part_index}" if doc.doc_id else None
+                        ),
+                        embedding=doc.embedding,
+                        extra_info={
+                            **(doc.extra_info or {}),
+                            "token_count": self.counter.count(child_text),
+                            "parent_text": parent_text,
+                        },
+                    )
+                    processed.append(child)
+                    part_index += 1
+        return processed
+
+
+class SemanticChunker(_BaseStrategyChunker):
+    """Group adjacent sentences by embedding similarity into coherent chunks.
+
+    Sentences are embedded in one batched call and split where the cosine
+    distance between consecutive sentences crosses a high percentile, so
+    chunk boundaries fall at topic shifts. Chunks are then token-capped at
+    ``max_tokens`` and merged up to clear ``min_tokens``. Any failure (too
+    few sentences, embedding error, degenerate distances) falls back to
+    ``RecursiveChunker`` so ingest never crashes.
+    """
+
+    _SENTENCE = re.compile(r"(?<=[.!?])\s+")
+    _PERCENTILE = 95.0
+
+    def _split_sentences(self, text: str) -> List[str]:
+        return [s for s in (p.strip() for p in self._SENTENCE.split(text)) if s]
+
+    def _fallback(self, documents: List[Document]) -> List[Document]:
+        recursive = RecursiveChunker(
+            chunking_strategy=self.chunking_strategy,
+            max_tokens=self.max_tokens,
+            min_tokens=self.min_tokens,
+            duplicate_headers=self.duplicate_headers,
+        )
+        return recursive.chunk(documents)
+
+    def _breakpoints(self, embeddings) -> set:
+        """Indices after which a new chunk starts, from high-distance gaps."""
+        import numpy as np
+
+        matrix = np.asarray(embeddings, dtype=np.float64)
+        if matrix.ndim != 2 or matrix.shape[0] < 2:
+            return set()
+        norms = np.linalg.norm(matrix, axis=1)
+        norms[norms == 0] = 1.0
+        unit = matrix / norms[:, None]
+        sims = np.sum(unit[:-1] * unit[1:], axis=1)
+        distances = 1.0 - sims
+        if not np.all(np.isfinite(distances)):
+            raise ValueError("non-finite semantic distances")
+        if float(distances.max()) <= float(distances.min()):
+            return set()
+        threshold = np.percentile(distances, self._PERCENTILE)
+        return {int(i) for i in np.where(distances >= threshold)[0]}
+
+    def _group(self, sentences: List[str], breakpoints: set) -> List[str]:
+        groups: List[str] = []
+        current: List[str] = []
+        for idx, sentence in enumerate(sentences):
+            current.append(sentence)
+            if idx in breakpoints:
+                groups.append(" ".join(current))
+                current = []
+        if current:
+            groups.append(" ".join(current))
+        return groups
+
+    def _enforce_tokens(self, groups: List[str]) -> List[str]:
+        """Hard-split groups over ``max_tokens`` then merge to clear min."""
+        capped: List[str] = []
+        for group in groups:
+            if self._token_count(group) <= self.max_tokens:
+                capped.append(group)
+            else:
+                capped.extend(p for p in self._split_by_tokens(group) if p.strip())
+        return self._merge_to_min(capped, " ")
+
+    def _chunk_text(self, text: str) -> List[str]:
+        sentences = self._split_sentences(text)
+        if len(sentences) < 2:
+            raise ValueError("too few sentences for semantic chunking")
+        from docsgpt.vectorstore.base import get_embeddings
+
+        embeddings = get_embeddings().embed_documents(sentences)
+        breakpoints = self._breakpoints(embeddings)
+        groups = self._group(sentences, breakpoints)
+        return [g for g in self._enforce_tokens(groups) if g.strip()]
+
+    def chunk(self, documents: List[Document]) -> List[Document]:
+        processed: List[Document] = []
+        for doc in documents:
+            try:
+                texts = self._chunk_text(doc.text)
+            except Exception as exc:
+                logger.warning(
+                    "Semantic chunking failed (%s); falling back to recursive.",
+                    exc,
+                )
+                processed.extend(self._fallback([doc]))
+                continue
+            for idx, text in enumerate(texts):
+                processed.append(self._emit(doc, idx, text))
+        return processed
+
+
+ChunkerCreator.register("recursive", RecursiveChunker)
+ChunkerCreator.register("markdown", MarkdownChunker)
+ChunkerCreator.register("parent_child", ParentChildChunker)
+ChunkerCreator.register("semantic", SemanticChunker)
+
+# Reuse the classic Chunker reference so this module can be the single import
+# that pulls every strategy into the registry.
+__all__ = [
+    "RecursiveChunker",
+    "MarkdownChunker",
+    "ParentChildChunker",
+    "SemanticChunker",
+    "Chunker",
+]

docsgpt/parser/connectors/__init__.py

@@ -0,0 +1,18 @@
+"""
+External knowledge base connectors for DocsGPT.
+
+This module contains connectors for external knowledge bases and document storage systems
+that require authentication and specialized handling, separate from simple web scrapers.
+"""
+
+from .base import BaseConnectorAuth, BaseConnectorLoader
+from .connector_creator import ConnectorCreator
+from .google_drive import GoogleDriveAuth, GoogleDriveLoader
+
+__all__ = [
+    'BaseConnectorAuth',
+    'BaseConnectorLoader',
+    'ConnectorCreator',
+    'GoogleDriveAuth',
+    'GoogleDriveLoader'
+]

docsgpt/parser/connectors/_auth_utils.py

@@ -0,0 +1,37 @@
+"""Shared helpers for connector auth modules.
+
+These helpers exist so that sensitive values (session tokens, bearer
+credentials) never end up interpolated into exception messages or log
+lines. Exception messages frequently flow into ``stack_logs`` (Postgres)
+and Sentry via ``exc_info=True``, so the raw value must never be the
+thing we format.
+"""
+
+from __future__ import annotations
+
+import hashlib
+
+
+def session_token_fingerprint(session_token: str) -> str:
+    """Return a short, irreversible fingerprint for a session token.
+
+    The returned string is safe to embed in exception messages and log
+    lines: it is a prefix of a SHA-256 digest, clearly tagged so an
+    operator reading the log knows it is a hash and not the token
+    itself. It is stable for a given input, which lets operators
+    correlate "which token failed" across log lines without exposing
+    the credential.
+
+    Args:
+        session_token: The raw session token. Accepts ``None`` or the
+            empty string for defensive callers; both yield a distinct
+            sentinel rather than raising.
+
+    Returns:
+        A string of the form ``"sha256:<6 hex chars>"``, or
+        ``"sha256:<empty>"`` when the input is falsy.
+    """
+    if not session_token:
+        return "sha256:<empty>"
+    digest = hashlib.sha256(session_token.encode("utf-8")).hexdigest()
+    return f"sha256:{digest[:6]}"

docsgpt/parser/connectors/base.py

@@ -0,0 +1,140 @@
+"""
+Base classes for external knowledge base connectors.
+
+This module provides minimal abstract base classes that define the essential
+interface for external knowledge base connectors.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional
+
+from docsgpt.parser.schema.base import Document
+
+
+class BaseConnectorAuth(ABC):
+    """
+    Abstract base class for connector authentication.
+    
+    Defines the minimal interface that all connector authentication
+    implementations must follow.
+    """
+    
+    @abstractmethod
+    def get_authorization_url(self, state: Optional[str] = None) -> str:
+        """
+        Generate authorization URL for OAuth flows.
+        
+        Args:
+            state: Optional state parameter for CSRF protection
+            
+        Returns:
+            Authorization URL
+        """
+        pass
+    
+    @abstractmethod
+    def exchange_code_for_tokens(self, authorization_code: str) -> Dict[str, Any]:
+        """
+        Exchange authorization code for access tokens.
+        
+        Args:
+            authorization_code: Authorization code from OAuth callback
+            
+        Returns:
+            Dictionary containing token information
+        """
+        pass
+    
+    @abstractmethod
+    def refresh_access_token(self, refresh_token: str) -> Dict[str, Any]:
+        """
+        Refresh an expired access token.
+        
+        Args:
+            refresh_token: Refresh token
+            
+        Returns:
+            Dictionary containing refreshed token information
+        """
+        pass
+    
+    @abstractmethod
+    def is_token_expired(self, token_info: Dict[str, Any]) -> bool:
+        """
+        Check if a token is expired.
+
+        Args:
+            token_info: Token information dictionary
+
+        Returns:
+            True if token is expired, False otherwise
+        """
+        pass
+
+    def sanitize_token_info(self, token_info: Dict[str, Any], **extra_fields) -> Dict[str, Any]:
+        """Extract the fields safe to persist in the session store.
+        """
+        return {
+            "access_token": token_info.get("access_token"),
+            "refresh_token": token_info.get("refresh_token"),
+            "token_uri": token_info.get("token_uri"),
+            "expiry": token_info.get("expiry"),
+            **extra_fields,
+        }
+
+
+class BaseConnectorLoader(ABC):
+    """
+    Abstract base class for connector loaders.
+    
+    Defines the minimal interface that all connector loader
+    implementations must follow.
+    """
+    
+    @abstractmethod
+    def __init__(self, session_token: str):
+        """
+        Initialize the connector loader.
+        
+        Args:
+            session_token: Authentication session token
+        """
+        pass
+    
+    @abstractmethod
+    def load_data(self, inputs: Dict[str, Any]) -> List[Document]:
+        """
+        Load documents from the external knowledge base.
+        
+        Args:
+            inputs: Configuration dictionary containing:
+                - file_ids: Optional list of specific file IDs to load
+                - folder_ids: Optional list of folder IDs to browse/download
+                - limit: Maximum number of items to return
+                - list_only: If True, return metadata without content
+                - recursive: Whether to recursively process folders
+                
+        Returns:
+            List of Document objects
+        """
+        pass
+    
+    @abstractmethod
+    def download_to_directory(self, local_dir: str, source_config: Dict[str, Any] = None) -> Dict[str, Any]:
+        """
+        Download files/folders to a local directory.
+        
+        Args:
+            local_dir: Local directory path to download files to
+            source_config: Configuration for what to download
+            
+        Returns:
+            Dictionary containing download results:
+                - files_downloaded: Number of files downloaded
+                - directory_path: Path where files were downloaded
+                - empty_result: Whether no files were downloaded
+                - source_type: Type of connector
+                - config_used: Configuration that was used
+                - error: Error message if download failed (optional)
+        """
+        pass

docsgpt/parser/connectors/confluence/__init__.py

@@ -0,0 +1,4 @@
+from .auth import ConfluenceAuth
+from .loader import ConfluenceLoader
+
+__all__ = ["ConfluenceAuth", "ConfluenceLoader"]

docsgpt/parser/connectors/confluence/auth.py

@@ -0,0 +1,221 @@
+import datetime
+import logging
+from typing import Any, Dict, Optional
+from urllib.parse import urlencode
+
+import requests
+
+from docsgpt.core.settings import settings
+from docsgpt.parser.connectors._auth_utils import session_token_fingerprint
+from docsgpt.parser.connectors.base import BaseConnectorAuth
+
+logger = logging.getLogger(__name__)
+
+
+class ConfluenceAuth(BaseConnectorAuth):
+
+    SCOPES = [
+        "read:page:confluence",
+        "read:space:confluence",
+        "read:attachment:confluence",
+        "read:me",
+        "offline_access",
+    ]
+
+    AUTH_URL = "https://auth.atlassian.com/authorize"
+    TOKEN_URL = "https://auth.atlassian.com/oauth/token"
+    RESOURCES_URL = "https://api.atlassian.com/oauth/token/accessible-resources"
+    ME_URL = "https://api.atlassian.com/me"
+
+    def __init__(self):
+        self.client_id = settings.CONFLUENCE_CLIENT_ID
+        self.client_secret = settings.CONFLUENCE_CLIENT_SECRET
+        self.redirect_uri = settings.CONNECTOR_REDIRECT_BASE_URI
+
+        if not self.client_id or not self.client_secret:
+            raise ValueError(
+                "Confluence OAuth credentials not configured. "
+                "Please set CONFLUENCE_CLIENT_ID and CONFLUENCE_CLIENT_SECRET in settings."
+            )
+
+    def get_authorization_url(self, state: Optional[str] = None) -> str:
+        params = {
+            "audience": "api.atlassian.com",
+            "client_id": self.client_id,
+            "scope": " ".join(self.SCOPES),
+            "redirect_uri": self.redirect_uri,
+            "state": state,
+            "response_type": "code",
+            "prompt": "consent",
+        }
+        return f"{self.AUTH_URL}?{urlencode(params)}"
+
+    def exchange_code_for_tokens(self, authorization_code: str) -> Dict[str, Any]:
+        if not authorization_code:
+            raise ValueError("Authorization code is required")
+
+        response = requests.post(
+            self.TOKEN_URL,
+            json={
+                "grant_type": "authorization_code",
+                "client_id": self.client_id,
+                "client_secret": self.client_secret,
+                "code": authorization_code,
+                "redirect_uri": self.redirect_uri,
+            },
+            headers={"Content-Type": "application/json"},
+            timeout=30,
+        )
+        response.raise_for_status()
+        token_data = response.json()
+
+        access_token = token_data.get("access_token")
+        if not access_token:
+            raise ValueError("OAuth flow did not return an access token")
+
+        refresh_token = token_data.get("refresh_token")
+        if not refresh_token:
+            raise ValueError("OAuth flow did not return a refresh token")
+
+        expires_in = token_data.get("expires_in", 3600)
+        expiry = (
+            datetime.datetime.now(datetime.timezone.utc)
+            + datetime.timedelta(seconds=expires_in)
+        ).isoformat()
+
+        cloud_id = self._fetch_cloud_id(access_token)
+        user_info = self._fetch_user_info(access_token)
+
+        return {
+            "access_token": access_token,
+            "refresh_token": refresh_token,
+            "token_uri": self.TOKEN_URL,
+            "scopes": self.SCOPES,
+            "expiry": expiry,
+            "cloud_id": cloud_id,
+            "user_info": {
+                "name": user_info.get("display_name", ""),
+                "email": user_info.get("email", ""),
+            },
+        }
+
+    def refresh_access_token(self, refresh_token: str) -> Dict[str, Any]:
+        if not refresh_token:
+            raise ValueError("Refresh token is required")
+
+        response = requests.post(
+            self.TOKEN_URL,
+            json={
+                "grant_type": "refresh_token",
+                "client_id": self.client_id,
+                "client_secret": self.client_secret,
+                "refresh_token": refresh_token,
+            },
+            headers={"Content-Type": "application/json"},
+            timeout=30,
+        )
+        response.raise_for_status()
+        token_data = response.json()
+
+        access_token = token_data.get("access_token")
+        new_refresh_token = token_data.get("refresh_token", refresh_token)
+
+        expires_in = token_data.get("expires_in", 3600)
+        expiry = (
+            datetime.datetime.now(datetime.timezone.utc)
+            + datetime.timedelta(seconds=expires_in)
+        ).isoformat()
+
+        cloud_id = self._fetch_cloud_id(access_token)
+
+        return {
+            "access_token": access_token,
+            "refresh_token": new_refresh_token,
+            "token_uri": self.TOKEN_URL,
+            "scopes": self.SCOPES,
+            "expiry": expiry,
+            "cloud_id": cloud_id,
+        }
+
+    def is_token_expired(self, token_info: Dict[str, Any]) -> bool:
+        if not token_info:
+            return True
+
+        expiry = token_info.get("expiry")
+        if not expiry:
+            return bool(token_info.get("access_token"))
+
+        try:
+            expiry_dt = datetime.datetime.fromisoformat(expiry)
+            now = datetime.datetime.now(datetime.timezone.utc)
+            return now >= expiry_dt - datetime.timedelta(seconds=60)
+        except Exception:
+            return True
+
+    def get_token_info_from_session(self, session_token: str) -> Dict[str, Any]:
+        from docsgpt.storage.db.repositories.connector_sessions import (
+            ConnectorSessionsRepository,
+        )
+        from docsgpt.storage.db.session import db_readonly
+
+        with db_readonly() as conn:
+            session = ConnectorSessionsRepository(conn).get_by_session_token(
+                session_token
+            )
+        if not session:
+            raise ValueError(
+                f"Invalid session token ({session_token_fingerprint(session_token)})"
+            )
+
+        token_info = session.get("token_info")
+        if not token_info:
+            raise ValueError("Session missing token information")
+
+        required = ["access_token", "refresh_token", "cloud_id"]
+        missing = [f for f in required if not token_info.get(f)]
+        if missing:
+            raise ValueError(f"Missing required token fields: {missing}")
+
+        return token_info
+
+    def sanitize_token_info(
+        self, token_info: Dict[str, Any], **extra_fields
+    ) -> Dict[str, Any]:
+        return super().sanitize_token_info(
+            token_info,
+            cloud_id=token_info.get("cloud_id"),
+            **extra_fields,
+        )
+
+    def _fetch_cloud_id(self, access_token: str) -> str:
+        response = requests.get(
+            self.RESOURCES_URL,
+            headers={
+                "Authorization": f"Bearer {access_token}",
+                "Accept": "application/json",
+            },
+            timeout=30,
+        )
+        response.raise_for_status()
+        resources = response.json()
+
+        if not resources:
+            raise ValueError("No accessible Confluence sites found for this account")
+
+        return resources[0]["id"]
+
+    def _fetch_user_info(self, access_token: str) -> Dict[str, Any]:
+        try:
+            response = requests.get(
+                self.ME_URL,
+                headers={
+                    "Authorization": f"Bearer {access_token}",
+                    "Accept": "application/json",
+                },
+                timeout=30,
+            )
+            response.raise_for_status()
+            return response.json()
+        except Exception as e:
+            logger.warning("Could not fetch user info: %s", e)
+            return {}

docsgpt/parser/connectors/confluence/loader.py

@@ -0,0 +1,417 @@
+import functools
+import logging
+import os
+from typing import Any, Dict, List, Optional
+
+import requests
+
+from docsgpt.parser.connectors.base import BaseConnectorLoader
+from docsgpt.parser.connectors.confluence.auth import ConfluenceAuth
+from docsgpt.parser.schema.base import Document
+
+logger = logging.getLogger(__name__)
+
+API_V2 = "https://api.atlassian.com/ex/confluence/{cloud_id}/wiki/api/v2"
+DOWNLOAD_BASE = "https://api.atlassian.com/ex/confluence/{cloud_id}/wiki"
+
+SUPPORTED_ATTACHMENT_TYPES = {
+    "application/pdf": ".pdf",
+    "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
+    "application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx",
+    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",
+    "application/msword": ".doc",
+    "application/vnd.ms-powerpoint": ".ppt",
+    "application/vnd.ms-excel": ".xls",
+    "text/plain": ".txt",
+    "text/csv": ".csv",
+    "text/html": ".html",
+    "text/markdown": ".md",
+    "application/json": ".json",
+    "application/epub+zip": ".epub",
+    "image/jpeg": ".jpg",
+    "image/png": ".png",
+}
+
+
+def _retry_on_auth_failure(func):
+    @functools.wraps(func)
+    def wrapper(self, *args, **kwargs):
+        try:
+            return func(self, *args, **kwargs)
+        except requests.exceptions.HTTPError as e:
+            if e.response is not None and e.response.status_code in (401, 403):
+                logger.info(
+                    "Auth failure in %s, refreshing token and retrying", func.__name__
+                )
+                try:
+                    new_token_info = self.auth.refresh_access_token(self.refresh_token)
+                    self.access_token = new_token_info["access_token"]
+                    self.refresh_token = new_token_info.get(
+                        "refresh_token", self.refresh_token
+                    )
+                    self._persist_refreshed_tokens(new_token_info)
+                except Exception as refresh_err:
+                    raise ValueError(
+                        f"Authentication failed and could not be refreshed: {refresh_err}"
+                    ) from e
+                return func(self, *args, **kwargs)
+            raise
+
+    return wrapper
+
+
+class ConfluenceLoader(BaseConnectorLoader):
+
+    def __init__(self, session_token: str):
+        self.auth = ConfluenceAuth()
+        self.session_token = session_token
+
+        token_info = self.auth.get_token_info_from_session(session_token)
+        self.access_token = token_info["access_token"]
+        self.refresh_token = token_info["refresh_token"]
+        self.cloud_id = token_info["cloud_id"]
+
+        self.base_url = API_V2.format(cloud_id=self.cloud_id)
+        self.download_base = DOWNLOAD_BASE.format(cloud_id=self.cloud_id)
+        self.next_page_token = None
+
+    def _headers(self) -> Dict[str, str]:
+        return {
+            "Authorization": f"Bearer {self.access_token}",
+            "Accept": "application/json",
+        }
+
+    def _persist_refreshed_tokens(self, token_info: Dict[str, Any]) -> None:
+        try:
+            from docsgpt.storage.db.repositories.connector_sessions import (
+                ConnectorSessionsRepository,
+            )
+            from docsgpt.storage.db.session import db_session
+
+            sanitized = self.auth.sanitize_token_info(token_info)
+            with db_session() as conn:
+                repo = ConnectorSessionsRepository(conn)
+                session = repo.get_by_session_token(self.session_token)
+                if session:
+                    repo.update(str(session["id"]), {"token_info": sanitized})
+        except Exception as e:
+            logger.warning("Failed to persist refreshed tokens: %s", e)
+
+    @_retry_on_auth_failure
+    def load_data(self, inputs: Dict[str, Any]) -> List[Document]:
+        folder_id = inputs.get("folder_id")
+        file_ids = inputs.get("file_ids", [])
+        limit = inputs.get("limit", 100)
+        list_only = inputs.get("list_only", False)
+        page_token = inputs.get("page_token")
+        search_query = inputs.get("search_query")
+        self.next_page_token = None
+
+        if file_ids:
+            return self._load_pages_by_ids(file_ids, list_only, search_query)
+
+        if folder_id:
+            return self._list_pages_in_space(
+                folder_id, limit, list_only, page_token, search_query
+            )
+
+        return self._list_spaces(limit, page_token, search_query)
+
+    @_retry_on_auth_failure
+    def download_to_directory(self, local_dir: str, source_config: dict = None) -> dict:
+        config = source_config or getattr(self, "config", {})
+        file_ids = config.get("file_ids", [])
+        folder_ids = config.get("folder_ids", [])
+        files_downloaded = 0
+
+        os.makedirs(local_dir, exist_ok=True)
+
+        if isinstance(file_ids, str):
+            file_ids = [file_ids]
+        if isinstance(folder_ids, str):
+            folder_ids = [folder_ids]
+
+        for page_id in file_ids:
+            if self._download_page(page_id, local_dir):
+                files_downloaded += 1
+            files_downloaded += self._download_page_attachments(page_id, local_dir)
+
+        for space_id in folder_ids:
+            files_downloaded += self._download_space(space_id, local_dir)
+
+        return {
+            "files_downloaded": files_downloaded,
+            "directory_path": local_dir,
+            "empty_result": files_downloaded == 0,
+            "source_type": "confluence",
+            "config_used": config,
+        }
+
+    def _list_spaces(
+        self, limit: int, cursor: Optional[str], search_query: Optional[str]
+    ) -> List[Document]:
+        documents: List[Document] = []
+        params: Dict[str, Any] = {"limit": min(limit, 250)}
+        if cursor:
+            params["cursor"] = cursor
+
+        response = requests.get(
+            f"{self.base_url}/spaces",
+            headers=self._headers(),
+            params=params,
+            timeout=30,
+        )
+        response.raise_for_status()
+        data = response.json()
+
+        for space in data.get("results", []):
+            name = space.get("name", "")
+            if search_query and search_query.lower() not in name.lower():
+                continue
+
+            documents.append(
+                Document(
+                    text="",
+                    doc_id=space["id"],
+                    extra_info={
+                        "file_name": name,
+                        "mime_type": "folder",
+                        "size": None,
+                        "created_time": space.get("createdAt"),
+                        "modified_time": None,
+                        "source": "confluence",
+                        "is_folder": True,
+                        "space_key": space.get("key"),
+                    },
+                )
+            )
+
+        next_link = data.get("_links", {}).get("next")
+        self.next_page_token = self._extract_cursor(next_link)
+        return documents
+
+    def _list_pages_in_space(
+        self,
+        space_id: str,
+        limit: int,
+        list_only: bool,
+        cursor: Optional[str],
+        search_query: Optional[str],
+    ) -> List[Document]:
+        documents: List[Document] = []
+        params: Dict[str, Any] = {"limit": min(limit, 250)}
+        if cursor:
+            params["cursor"] = cursor
+
+        response = requests.get(
+            f"{self.base_url}/spaces/{space_id}/pages",
+            headers=self._headers(),
+            params=params,
+            timeout=30,
+        )
+        response.raise_for_status()
+        data = response.json()
+
+        for page in data.get("results", []):
+            title = page.get("title", "")
+            if search_query and search_query.lower() not in title.lower():
+                continue
+
+            doc = self._page_to_document(
+                page, load_content=not list_only, space_id=space_id
+            )
+            if doc:
+                documents.append(doc)
+
+        next_link = data.get("_links", {}).get("next")
+        self.next_page_token = self._extract_cursor(next_link)
+        return documents
+
+    def _load_pages_by_ids(
+        self, page_ids: List[str], list_only: bool, search_query: Optional[str]
+    ) -> List[Document]:
+        documents: List[Document] = []
+        for page_id in page_ids:
+            try:
+                params: Dict[str, str] = {}
+                if not list_only:
+                    params["body-format"] = "storage"
+
+                response = requests.get(
+                    f"{self.base_url}/pages/{page_id}",
+                    headers=self._headers(),
+                    params=params,
+                    timeout=30,
+                )
+                response.raise_for_status()
+                page = response.json()
+
+                title = page.get("title", "")
+                if search_query and search_query.lower() not in title.lower():
+                    continue
+
+                doc = self._page_to_document(page, load_content=not list_only)
+                if doc:
+                    documents.append(doc)
+            except Exception as e:
+                logger.error("Error loading page %s: %s", page_id, e)
+        return documents
+
+    def _page_to_document(
+        self,
+        page: Dict[str, Any],
+        load_content: bool = False,
+        space_id: Optional[str] = None,
+    ) -> Optional[Document]:
+        page_id = page.get("id")
+        title = page.get("title", "Unknown")
+        version = page.get("version", {})
+        modified_time = version.get("createdAt") if isinstance(version, dict) else None
+        created_time = page.get("createdAt")
+        resolved_space_id = space_id or page.get("spaceId")
+
+        text = ""
+        if load_content:
+            body = page.get("body", {})
+            storage = body.get("storage", {}) if isinstance(body, dict) else {}
+            text = storage.get("value", "") if isinstance(storage, dict) else ""
+
+        return Document(
+            text=text,
+            doc_id=str(page_id),
+            extra_info={
+                "file_name": title,
+                "mime_type": "text/html",
+                "size": len(text) if text else None,
+                "created_time": created_time,
+                "modified_time": modified_time,
+                "source": "confluence",
+                "is_folder": False,
+                "page_id": str(page_id),
+                "space_id": resolved_space_id,
+                "cloud_id": self.cloud_id,
+            },
+        )
+
+    def _download_page(self, page_id: str, local_dir: str) -> bool:
+        try:
+            response = requests.get(
+                f"{self.base_url}/pages/{page_id}",
+                headers=self._headers(),
+                params={"body-format": "storage"},
+                timeout=30,
+            )
+            response.raise_for_status()
+            page = response.json()
+
+            title = page.get("title", page_id)
+            safe_name = "".join(c if c.isalnum() or c in " -_" else "_" for c in title)
+            body = page.get("body", {}).get("storage", {}).get("value", "")
+
+            file_path = os.path.join(local_dir, f"{safe_name}.html")
+            with open(file_path, "w", encoding="utf-8") as f:
+                f.write(body)
+
+            return True
+        except Exception as e:
+            logger.error("Error downloading page %s: %s", page_id, e)
+            return False
+
+    def _download_page_attachments(self, page_id: str, local_dir: str) -> int:
+        downloaded = 0
+        try:
+            cursor = None
+            while True:
+                params: Dict[str, Any] = {"limit": 100}
+                if cursor:
+                    params["cursor"] = cursor
+
+                response = requests.get(
+                    f"{self.base_url}/pages/{page_id}/attachments",
+                    headers=self._headers(),
+                    params=params,
+                    timeout=30,
+                )
+                response.raise_for_status()
+                data = response.json()
+
+                for att in data.get("results", []):
+                    media_type = att.get("mediaType", "")
+                    if media_type not in SUPPORTED_ATTACHMENT_TYPES:
+                        continue
+
+                    download_link = att.get("_links", {}).get("download")
+                    if not download_link:
+                        continue
+
+                    raw_name = att.get("title", att.get("id", "attachment"))
+                    file_name = "".join(
+                        c if c.isalnum() or c in " -_." else "_"
+                        for c in os.path.basename(raw_name)
+                    ) or "attachment"
+                    file_path = os.path.join(local_dir, file_name)
+
+                    url = f"{self.download_base}{download_link}"
+                    file_resp = requests.get(
+                        url, headers=self._headers(), timeout=60, stream=True
+                    )
+                    file_resp.raise_for_status()
+
+                    with open(file_path, "wb") as f:
+                        for chunk in file_resp.iter_content(chunk_size=8192):
+                            f.write(chunk)
+
+                    downloaded += 1
+
+                next_link = data.get("_links", {}).get("next")
+                cursor = self._extract_cursor(next_link)
+                if not cursor:
+                    break
+
+        except Exception as e:
+            logger.error("Error downloading attachments for page %s: %s", page_id, e)
+        return downloaded
+
+    def _download_space(self, space_id: str, local_dir: str) -> int:
+        downloaded = 0
+        cursor = None
+        while True:
+            params: Dict[str, Any] = {"limit": 250}
+            if cursor:
+                params["cursor"] = cursor
+
+            try:
+                response = requests.get(
+                    f"{self.base_url}/spaces/{space_id}/pages",
+                    headers=self._headers(),
+                    params=params,
+                    timeout=30,
+                )
+                response.raise_for_status()
+                data = response.json()
+            except Exception as e:
+                logger.error("Error listing pages in space %s: %s", space_id, e)
+                break
+
+            for page in data.get("results", []):
+                page_id = page.get("id")
+                if self._download_page(str(page_id), local_dir):
+                    downloaded += 1
+                downloaded += self._download_page_attachments(str(page_id), local_dir)
+
+            next_link = data.get("_links", {}).get("next")
+            cursor = self._extract_cursor(next_link)
+            if not cursor:
+                break
+
+        return downloaded
+
+    @staticmethod
+    def _extract_cursor(next_link: Optional[str]) -> Optional[str]:
+        if not next_link:
+            return None
+        from urllib.parse import parse_qs, urlparse
+
+        parsed = urlparse(next_link)
+        cursors = parse_qs(parsed.query).get("cursor")
+        return cursors[0] if cursors else None

docsgpt/parser/connectors/connector_creator.py

@@ -0,0 +1,89 @@
+from docsgpt.parser.connectors.confluence.auth import ConfluenceAuth
+from docsgpt.parser.connectors.confluence.loader import ConfluenceLoader
+from docsgpt.parser.connectors.google_drive.auth import GoogleDriveAuth
+from docsgpt.parser.connectors.google_drive.loader import GoogleDriveLoader
+from docsgpt.parser.connectors.share_point.auth import SharePointAuth
+from docsgpt.parser.connectors.share_point.loader import SharePointLoader
+
+
+class ConnectorCreator:
+    """
+    Factory class for creating external knowledge base connectors and auth providers.
+
+    These are different from remote loaders as they typically require
+    authentication and connect to external document storage systems.
+    """
+
+    connectors = {
+        "confluence": ConfluenceLoader,
+        "google_drive": GoogleDriveLoader,
+        "share_point": SharePointLoader,
+    }
+
+    auth_providers = {
+        "confluence": ConfluenceAuth,
+        "google_drive": GoogleDriveAuth,
+        "share_point": SharePointAuth,
+    }
+
+    @classmethod
+    def create_connector(cls, connector_type, *args, **kwargs):
+        """
+        Create a connector instance for the specified type.
+
+        Args:
+            connector_type: Type of connector to create (e.g., 'google_drive')
+            *args, **kwargs: Arguments to pass to the connector constructor
+
+        Returns:
+            Connector instance
+
+        Raises:
+            ValueError: If connector type is not supported
+        """
+        connector_class = cls.connectors.get(connector_type.lower())
+        if not connector_class:
+            raise ValueError(f"No connector class found for type {connector_type}")
+        return connector_class(*args, **kwargs)
+
+    @classmethod
+    def create_auth(cls, connector_type):
+        """
+        Create an auth provider instance for the specified connector type.
+
+        Args:
+            connector_type: Type of connector auth to create (e.g., 'google_drive')
+
+        Returns:
+            Auth provider instance
+
+        Raises:
+            ValueError: If connector type is not supported for auth
+        """
+        auth_class = cls.auth_providers.get(connector_type.lower())
+        if not auth_class:
+            raise ValueError(f"No auth class found for type {connector_type}")
+        return auth_class()
+
+    @classmethod
+    def get_supported_connectors(cls):
+        """
+        Get list of supported connector types.
+
+        Returns:
+            List of supported connector type strings
+        """
+        return list(cls.connectors.keys())
+
+    @classmethod
+    def is_supported(cls, connector_type):
+        """
+        Check if a connector type is supported.
+
+        Args:
+            connector_type: Type of connector to check
+
+        Returns:
+            True if supported, False otherwise
+        """
+        return connector_type.lower() in cls.connectors

docsgpt/parser/connectors/google_drive/__init__.py

@@ -0,0 +1,10 @@
+"""
+Google Drive connector for DocsGPT.
+
+This module provides authentication and document loading capabilities for Google Drive.
+"""
+
+from .auth import GoogleDriveAuth
+from .loader import GoogleDriveLoader
+
+__all__ = ['GoogleDriveAuth', 'GoogleDriveLoader']

docsgpt/parser/connectors/google_drive/auth.py

@@ -0,0 +1,269 @@
+import logging
+import datetime
+from typing import Optional, Dict, Any
+
+from google.oauth2.credentials import Credentials
+from google_auth_oauthlib.flow import Flow
+from googleapiclient.discovery import build
+from googleapiclient.errors import HttpError
+
+from docsgpt.core.settings import settings
+from docsgpt.parser.connectors._auth_utils import session_token_fingerprint
+from docsgpt.parser.connectors.base import BaseConnectorAuth
+
+
+class GoogleDriveAuth(BaseConnectorAuth):
+    """
+    Handles Google OAuth 2.0 authentication for Google Drive access.
+    """
+    
+    SCOPES = [
+        'https://www.googleapis.com/auth/drive.readonly'
+    ]
+    
+    def __init__(self):
+        self.client_id = settings.GOOGLE_CLIENT_ID
+        self.client_secret = settings.GOOGLE_CLIENT_SECRET
+        self.redirect_uri = f"{settings.CONNECTOR_REDIRECT_BASE_URI}"
+        
+        if not self.client_id or not self.client_secret:
+            raise ValueError("Google OAuth credentials not configured. Please set GOOGLE_CLIENT_ID and GOOGLE_CLIENT_SECRET in settings.")
+
+
+
+    def get_authorization_url(self, state: Optional[str] = None) -> str:
+        try:
+            flow = Flow.from_client_config(
+                {
+                    "web": {
+                        "client_id": self.client_id,
+                        "client_secret": self.client_secret,
+                        "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+                        "token_uri": "https://oauth2.googleapis.com/token",
+                        "redirect_uris": [self.redirect_uri]
+                    }
+                },
+                scopes=self.SCOPES,
+                # Flow is rebuilt at exchange time, so an auto-generated
+                # code_verifier wouldn't survive; confidential client doesn't
+                # need PKCE anyway.
+                autogenerate_code_verifier=False,
+            )
+            flow.redirect_uri = self.redirect_uri
+
+            authorization_url, _ = flow.authorization_url(
+                access_type='offline',
+                prompt='consent',
+                include_granted_scopes='false',
+                state=state
+            )
+
+            return authorization_url
+
+        except Exception as e:
+            logging.error(f"Error generating authorization URL: {e}")
+            raise
+
+    def exchange_code_for_tokens(self, authorization_code: str) -> Dict[str, Any]:
+        try:
+            if not authorization_code:
+                raise ValueError("Authorization code is required")
+
+            flow = Flow.from_client_config(
+                {
+                    "web": {
+                        "client_id": self.client_id,
+                        "client_secret": self.client_secret,
+                        "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+                        "token_uri": "https://oauth2.googleapis.com/token",
+                        "redirect_uris": [self.redirect_uri]
+                    }
+                },
+                scopes=self.SCOPES,
+                autogenerate_code_verifier=False,
+            )
+            flow.redirect_uri = self.redirect_uri
+
+            flow.fetch_token(code=authorization_code)
+
+            credentials = flow.credentials
+
+            if not credentials.refresh_token:
+                logging.warning("OAuth flow did not return a refresh_token.")
+            if not credentials.token:
+                raise ValueError("OAuth flow did not return an access token")
+
+            if not credentials.token_uri:
+                credentials.token_uri = "https://oauth2.googleapis.com/token"
+
+            if not credentials.client_id:
+                credentials.client_id = self.client_id
+
+            if not credentials.client_secret:
+                credentials.client_secret = self.client_secret
+
+            if not credentials.refresh_token:
+                raise ValueError(
+                    "No refresh token received. This typically happens when offline access wasn't granted. "
+                )
+
+            return {
+                'access_token': credentials.token,
+                'refresh_token': credentials.refresh_token,
+                'token_uri': credentials.token_uri,
+                'client_id': credentials.client_id,
+                'client_secret': credentials.client_secret,
+                'scopes': credentials.scopes,
+                'expiry': credentials.expiry.isoformat() if credentials.expiry else None
+            }
+
+        except Exception as e:
+            logging.error(f"Error exchanging code for tokens: {e}")
+            raise
+    
+    def refresh_access_token(self, refresh_token: str) -> Dict[str, Any]:
+        try:
+            if not refresh_token:
+                raise ValueError("Refresh token is required")
+
+            credentials = Credentials(
+                token=None,
+                refresh_token=refresh_token,
+                token_uri="https://oauth2.googleapis.com/token",
+                client_id=self.client_id,
+                client_secret=self.client_secret
+            )
+
+            from google.auth.transport.requests import Request
+            credentials.refresh(Request())
+
+            return {
+                'access_token': credentials.token,
+                'refresh_token': refresh_token,
+                'token_uri': credentials.token_uri,
+                'client_id': credentials.client_id,
+                'client_secret': credentials.client_secret,
+                'scopes': credentials.scopes,
+                'expiry': credentials.expiry.isoformat() if credentials.expiry else None
+            }
+        except Exception as e:
+            logging.error(f"Error refreshing access token: {e}", exc_info=True)
+            raise
+    
+    def create_credentials_from_token_info(self, token_info: Dict[str, Any]) -> Credentials:
+        from docsgpt.core.settings import settings
+
+        access_token = token_info.get('access_token')
+        if not access_token:
+            raise ValueError("No access token found in token_info")
+
+        credentials = Credentials(
+            token=access_token,
+            refresh_token=token_info.get('refresh_token'),
+            token_uri= 'https://oauth2.googleapis.com/token',
+            client_id=settings.GOOGLE_CLIENT_ID,
+            client_secret=settings.GOOGLE_CLIENT_SECRET,
+            scopes=token_info.get('scopes', ['https://www.googleapis.com/auth/drive.readonly'])
+        )
+
+        if not credentials.token:
+            raise ValueError("Credentials created without valid access token")
+
+        return credentials
+    
+    def build_drive_service(self, credentials: Credentials):
+        try:
+            if not credentials:
+                raise ValueError("No credentials provided")
+
+            if not credentials.token and not credentials.refresh_token:
+                raise ValueError("No access token or refresh token available. User must re-authorize with offline access.")
+
+            needs_refresh = credentials.expired or not credentials.token
+            if needs_refresh:
+                if credentials.refresh_token:
+                    try:
+                        from google.auth.transport.requests import Request
+                        credentials.refresh(Request())
+                    except Exception as refresh_error:
+                        raise ValueError(f"Failed to refresh credentials: {refresh_error}")
+                else:
+                    raise ValueError("No access token or refresh token available. User must re-authorize with offline access.")
+
+            return build('drive', 'v3', credentials=credentials)
+
+        except HttpError as e:
+            raise ValueError(f"Failed to build Google Drive service: HTTP {e.resp.status}")
+        except Exception as e:
+            raise ValueError(f"Failed to build Google Drive service: {str(e)}")
+        
+    def is_token_expired(self, token_info):
+        if 'expiry' in token_info and token_info['expiry']:
+            try:
+                from dateutil import parser
+                # Google Drive provides timezone-aware ISO8601 dates
+                expiry_dt = parser.parse(token_info['expiry'])
+                current_time = datetime.datetime.now(datetime.timezone.utc)
+                return current_time >= expiry_dt - datetime.timedelta(seconds=60)
+            except Exception:
+                return True
+
+        if 'access_token' in token_info and token_info['access_token']:
+            return False
+
+        return True
+    
+    def get_token_info_from_session(self, session_token: str) -> Dict[str, Any]:
+        try:
+            from docsgpt.storage.db.repositories.connector_sessions import (
+                ConnectorSessionsRepository,
+            )
+            from docsgpt.storage.db.session import db_readonly
+
+            with db_readonly() as conn:
+                session = ConnectorSessionsRepository(conn).get_by_session_token(
+                    session_token
+                )
+            if not session:
+                raise ValueError(
+                    f"Invalid session token ({session_token_fingerprint(session_token)})"
+                )
+
+            token_info = session.get("token_info")
+            if not token_info:
+                raise ValueError("Session missing token information")
+
+            required_fields = ["access_token", "refresh_token"]
+            missing_fields = [field for field in required_fields if field not in token_info or not token_info.get(field)]
+            if missing_fields:
+                raise ValueError(f"Missing required token fields: {missing_fields}")
+
+            if 'token_uri' not in token_info:
+                token_info['token_uri'] = 'https://oauth2.googleapis.com/token'
+
+            return token_info
+
+        except Exception as e:
+            raise ValueError(f"Failed to retrieve Google Drive token information: {str(e)}")
+
+    def validate_credentials(self, credentials: Credentials) -> bool:
+        """
+        Validate Google Drive credentials by making a test API call.
+
+        Args:
+            credentials: Google credentials object
+
+        Returns:
+            True if credentials are valid, False otherwise
+        """
+        try:
+            service = self.build_drive_service(credentials)
+            service.about().get(fields="user").execute()
+            return True
+
+        except HttpError as e:
+            logging.error(f"HTTP error validating credentials: {e}")
+            return False
+        except Exception as e:
+            logging.error(f"Error validating credentials: {e}")
+            return False

docsgpt/parser/connectors/google_drive/loader.py

@@ -0,0 +1,561 @@
+"""
+Google Drive loader for DocsGPT.
+Loads documents from Google Drive using Google Drive API.
+"""
+
+import io
+import logging
+import os
+from typing import List, Dict, Any, Optional
+
+from googleapiclient.http import MediaIoBaseDownload
+from googleapiclient.errors import HttpError
+
+from docsgpt.parser.connectors.base import BaseConnectorLoader
+from docsgpt.parser.connectors.google_drive.auth import GoogleDriveAuth
+from docsgpt.parser.schema.base import Document
+
+
+class GoogleDriveLoader(BaseConnectorLoader):
+
+    SUPPORTED_MIME_TYPES = {
+        'application/pdf': '.pdf',
+        'application/vnd.google-apps.document': '.docx',
+        'application/vnd.google-apps.presentation': '.pptx',
+        'application/vnd.google-apps.spreadsheet': '.xlsx',
+        'application/vnd.openxmlformats-officedocument.wordprocessingml.document': '.docx',
+        'application/vnd.openxmlformats-officedocument.presentationml.presentation': '.pptx',
+        'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': '.xlsx',
+        'application/msword': '.doc',
+        'application/vnd.ms-powerpoint': '.ppt',
+        'application/vnd.ms-excel': '.xls',
+        'text/plain': '.txt',
+        'text/csv': '.csv',
+        'text/html': '.html',
+        'text/markdown': '.md',
+        'text/x-rst': '.rst',
+        'application/json': '.json',
+        'application/epub+zip': '.epub',
+        'application/rtf': '.rtf',
+        'image/jpeg': '.jpg',
+        'image/jpg': '.jpg',
+        'image/png': '.png',
+    }
+
+    EXPORT_FORMATS = {
+        'application/vnd.google-apps.document': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+        'application/vnd.google-apps.presentation': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+        'application/vnd.google-apps.spreadsheet': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
+    }
+
+    def __init__(self, session_token: str):
+        self.auth = GoogleDriveAuth()
+        self.session_token = session_token
+
+        token_info = self.auth.get_token_info_from_session(session_token)
+        self.credentials = self.auth.create_credentials_from_token_info(token_info)
+
+        try:
+            self.service = self.auth.build_drive_service(self.credentials)
+        except Exception as e:
+            logging.warning(f"Could not build Google Drive service: {e}")
+            self.service = None
+
+        self.next_page_token = None
+
+
+
+    def _process_file(self, file_metadata: Dict[str, Any], load_content: bool = True) -> Optional[Document]:
+        try:
+            file_id = file_metadata.get('id')
+            file_name = file_metadata.get('name', 'Unknown')
+            mime_type = file_metadata.get('mimeType', 'application/octet-stream')
+
+            if mime_type not in self.SUPPORTED_MIME_TYPES and not mime_type.startswith('application/vnd.google-apps.'):
+                return None
+            if mime_type not in self.SUPPORTED_MIME_TYPES and not mime_type.startswith('application/vnd.google-apps.'):
+                logging.info(f"Skipping unsupported file type: {mime_type} for file {file_name}")
+                return None
+            # Google Drive provides timezone-aware ISO8601 dates
+            doc_metadata = {
+                'file_name': file_name,
+                'mime_type': mime_type,
+                'size': file_metadata.get('size', None),
+                'created_time': file_metadata.get('createdTime'),
+                'modified_time': file_metadata.get('modifiedTime'),
+                'parents': file_metadata.get('parents', []),
+                'source': 'google_drive'
+            }
+
+            if not load_content:
+                return Document(
+                    text="",
+                    doc_id=file_id,
+                    extra_info=doc_metadata
+                )
+
+            content = self._download_file_content(file_id, mime_type)
+            if content is None:
+                logging.warning(f"Could not load content for file {file_name} ({file_id})")
+                return None
+
+            return Document(
+                text=content,
+                doc_id=file_id,
+                extra_info=doc_metadata
+            )
+
+        except Exception as e:
+            logging.error(f"Error processing file: {e}")
+            return None
+
+    def load_data(self, inputs: Dict[str, Any]) -> List[Document]:
+        session_token = inputs.get('session_token')
+        if session_token and session_token != self.session_token:
+            logging.warning("Session token in inputs differs from loader's session token. Using loader's session token.")
+        self.config = inputs
+
+        try:
+            documents: List[Document] = []
+
+            folder_id = inputs.get('folder_id')
+            file_ids = inputs.get('file_ids', [])
+            limit = inputs.get('limit', 100)
+            list_only = inputs.get('list_only', False)
+            load_content = not list_only
+            page_token = inputs.get('page_token')
+            search_query = inputs.get('search_query')
+            self.next_page_token = None
+
+            if file_ids:
+                # Specific files requested: load them
+                for file_id in file_ids:
+                    try:
+                        doc = self._load_file_by_id(file_id, load_content=load_content)
+                        if doc:
+                            if not search_query or (
+                                search_query.lower() in doc.extra_info.get('file_name', '').lower()
+                            ):
+                                documents.append(doc)
+                        elif hasattr(self, '_credential_refreshed') and self._credential_refreshed:
+                            self._credential_refreshed = False
+                            logging.info(f"Retrying load of file {file_id} after credential refresh")
+                            doc = self._load_file_by_id(file_id, load_content=load_content)
+                            if doc and (
+                                not search_query or 
+                                search_query.lower() in doc.extra_info.get('file_name', '').lower()
+                            ):
+                                documents.append(doc)
+                    except Exception as e:
+                        logging.error(f"Error loading file {file_id}: {e}")
+                        continue
+            else:
+                # Browsing mode: list immediate children of provided folder or root
+                parent_id = folder_id if folder_id else 'root'
+                documents = self._list_items_in_parent(
+                    parent_id, 
+                    limit=limit, 
+                    load_content=load_content, 
+                    page_token=page_token,
+                    search_query=search_query
+                )
+
+            logging.info(f"Loaded {len(documents)} documents from Google Drive")
+            return documents
+
+        except Exception as e:
+            logging.error(f"Error loading data from Google Drive: {e}", exc_info=True)
+            raise
+
+
+
+    def _load_file_by_id(self, file_id: str, load_content: bool = True) -> Optional[Document]:
+        self._ensure_service()
+
+        try:
+            file_metadata = self.service.files().get(
+                fileId=file_id,
+                fields='id,name,mimeType,size,createdTime,modifiedTime,parents',
+                supportsAllDrives=True
+            ).execute()
+
+            return self._process_file(file_metadata, load_content=load_content)
+
+        except HttpError as e:
+            logging.error(f"HTTP error loading file {file_id}: {e.resp.status} - {e.content}")
+
+            if e.resp.status in [401, 403]:
+                if hasattr(self.credentials, 'refresh_token') and self.credentials.refresh_token:
+                    try:
+                        from google.auth.transport.requests import Request
+                        self.credentials.refresh(Request())
+                        self._ensure_service()
+                        return None
+                    except Exception as refresh_error:
+                        raise ValueError(f"Authentication failed and could not be refreshed: {refresh_error}")
+                else:
+                    raise ValueError("Authentication failed and cannot be refreshed: missing refresh_token")
+
+            return None
+        except Exception as e:
+            logging.error(f"Error loading file {file_id}: {e}")
+            return None
+
+
+    def _list_items_in_parent(self, parent_id: str, limit: int = 100, load_content: bool = False, page_token: Optional[str] = None, search_query: Optional[str] = None) -> List[Document]:
+        self._ensure_service()
+
+        documents: List[Document] = []
+
+        try:
+            query = f"'{parent_id}' in parents and trashed=false"
+
+            if search_query:
+                safe_search = search_query.replace("'", "\\'")
+                query += f" and name contains '{safe_search}'"
+
+            next_token_out: Optional[str] = None
+
+            while True:
+                page_size = 100
+                if limit:
+                    remaining = max(0, limit - len(documents))
+                    if remaining == 0:
+                        break
+                    page_size = min(100, remaining)
+
+                results = self.service.files().list(
+                    q=query,
+                    fields='nextPageToken,files(id,name,mimeType,size,createdTime,modifiedTime,parents)',
+                    pageToken=page_token,
+                    pageSize=page_size,
+                    orderBy='name',
+                    supportsAllDrives=True,
+                    includeItemsFromAllDrives=True
+                ).execute()
+
+                items = results.get('files', [])
+                for item in items:
+                    mime_type = item.get('mimeType')
+                    if mime_type == 'application/vnd.google-apps.folder':
+                        doc_metadata = {
+                            'file_name': item.get('name', 'Unknown'),
+                            'mime_type': mime_type,
+                            'size': item.get('size', None),
+                            'created_time': item.get('createdTime'),
+                            'modified_time': item.get('modifiedTime'),
+                            'parents': item.get('parents', []),
+                            'source': 'google_drive',
+                            'is_folder': True
+                        }
+                        documents.append(Document(text="", doc_id=item.get('id'), extra_info=doc_metadata))
+                    else:
+                        doc = self._process_file(item, load_content=load_content)
+                        if doc:
+                            documents.append(doc)
+
+                    if limit and len(documents) >= limit:
+                        self.next_page_token = results.get('nextPageToken')
+                        return documents
+
+                page_token = results.get('nextPageToken')
+                next_token_out = page_token
+                if not page_token:
+                    break
+
+            self.next_page_token = next_token_out
+            return documents
+        except Exception as e:
+            logging.error(f"Error listing items under parent {parent_id}: {e}")
+            return documents
+
+
+
+
+    def _download_file_content(self, file_id: str, mime_type: str) -> Optional[str]:
+        if not self.credentials.token:
+            logging.warning("No access token in credentials, attempting to refresh")
+            if hasattr(self.credentials, 'refresh_token') and self.credentials.refresh_token:
+                try:
+                    from google.auth.transport.requests import Request
+                    self.credentials.refresh(Request())
+                    logging.info("Credentials refreshed successfully")
+                    self._ensure_service()
+                except Exception as e:
+                    logging.error(f"Failed to refresh credentials: {e}")
+                    raise ValueError("Authentication failed and cannot be refreshed: missing or invalid refresh_token")
+            else:
+                logging.error("No access token and no refresh_token available")
+                raise ValueError("Authentication failed and cannot be refreshed: missing refresh_token")
+
+        if self.credentials.expired:
+            logging.warning("Credentials are expired, attempting to refresh")
+            if hasattr(self.credentials, 'refresh_token') and self.credentials.refresh_token:
+                try:
+                    from google.auth.transport.requests import Request
+                    self.credentials.refresh(Request())
+                    logging.info("Credentials refreshed successfully")
+                    self._ensure_service()
+                except Exception as e:
+                    logging.error(f"Failed to refresh expired credentials: {e}")
+                    raise ValueError("Authentication failed and cannot be refreshed: expired credentials")
+            else:
+                logging.error("Credentials expired and no refresh_token available")
+                raise ValueError("Authentication failed and cannot be refreshed: missing refresh_token")
+
+        try:
+            if mime_type in self.EXPORT_FORMATS:
+                export_mime_type = self.EXPORT_FORMATS[mime_type]
+                request = self.service.files().export_media(
+                    fileId=file_id,
+                    mimeType=export_mime_type
+                )
+            else:
+                request = self.service.files().get_media(fileId=file_id, supportsAllDrives=True)
+
+            file_io = io.BytesIO()
+            downloader = MediaIoBaseDownload(file_io, request)
+
+            done = False
+            while done is False:
+                try:
+                    _, done = downloader.next_chunk()
+                except HttpError as e:
+                    logging.error(f"HTTP error downloading file {file_id}: {e.resp.status} - {e.content}")
+                    return None
+                except Exception as e:
+                    logging.error(f"Error during download of file {file_id}: {e}")
+                    return None
+
+            content_bytes = file_io.getvalue()
+
+            try:
+                return content_bytes.decode('utf-8')
+            except UnicodeDecodeError:
+                logging.error(f"Could not decode file {file_id} as text")
+                return None
+
+        except HttpError as e:
+            logging.error(f"HTTP error downloading file {file_id}: {e.resp.status} - {e.content}")
+
+            if e.resp.status in [401, 403]:
+                logging.error(f"Authentication error downloading file {file_id}")
+
+                if hasattr(self.credentials, 'refresh_token') and self.credentials.refresh_token:
+                    logging.info(f"Attempting to refresh credentials for file {file_id}")
+                    try:
+                        from google.auth.transport.requests import Request
+                        self.credentials.refresh(Request())
+                        logging.info("Credentials refreshed successfully")
+                        self._credential_refreshed = True
+                        self._ensure_service()
+                        return None
+                    except Exception as refresh_error:
+                        logging.error(f"Error refreshing credentials: {refresh_error}")
+                        raise ValueError(f"Authentication failed and could not be refreshed: {refresh_error}")
+                else:
+                    logging.error("Cannot refresh credentials: missing refresh_token")
+                    raise ValueError("Authentication failed and cannot be refreshed: missing refresh_token")
+
+            return None
+        except Exception as e:
+            logging.error(f"Error downloading file {file_id}: {e}")
+            return None
+
+
+    def _download_file_to_directory(self, file_id: str, local_dir: str) -> bool:
+        try:
+            self._ensure_service()
+            return self._download_single_file(file_id, local_dir)
+        except Exception as e:
+            logging.error(f"Error downloading file {file_id}: {e}", exc_info=True)
+            return False
+
+    def _ensure_service(self):
+        if not self.service:
+            try:
+                self.service = self.auth.build_drive_service(self.credentials)
+            except Exception as e:
+                raise ValueError(f"Cannot access Google Drive: {e}")
+
+    def _download_single_file(self, file_id: str, local_dir: str) -> bool:
+        file_metadata = self.service.files().get(
+            fileId=file_id,
+            fields='name,mimeType',
+            supportsAllDrives=True
+        ).execute()
+
+        file_name = file_metadata['name']
+        mime_type = file_metadata['mimeType']
+
+        if mime_type not in self.SUPPORTED_MIME_TYPES and not mime_type.startswith('application/vnd.google-apps.'):
+            return False
+
+        os.makedirs(local_dir, exist_ok=True)
+        full_path = os.path.join(local_dir, file_name)
+
+        if mime_type in self.EXPORT_FORMATS:
+            export_mime_type = self.EXPORT_FORMATS[mime_type]
+            request = self.service.files().export_media(
+                fileId=file_id,
+                mimeType=export_mime_type
+            )
+            extension = self._get_extension_for_mime_type(export_mime_type)
+            if not full_path.endswith(extension):
+                full_path += extension
+        else:
+            request = self.service.files().get_media(fileId=file_id, supportsAllDrives=True)
+
+        with open(full_path, 'wb') as f:
+            downloader = MediaIoBaseDownload(f, request)
+            done = False
+            while not done:
+                _, done = downloader.next_chunk()
+
+        return True
+
+    def _download_folder_recursive(self, folder_id: str, local_dir: str, recursive: bool = True) -> int:
+        files_downloaded = 0
+        try:
+            os.makedirs(local_dir, exist_ok=True)
+
+            query = f"'{folder_id}' in parents and trashed=false"
+            page_token = None
+
+            while True:
+                results = self.service.files().list(
+                    q=query,
+                    fields='nextPageToken, files(id, name, mimeType)',
+                    pageToken=page_token,
+                    pageSize=1000,
+                    supportsAllDrives=True,
+                    includeItemsFromAllDrives=True
+                ).execute()
+
+                items = results.get('files', [])
+                logging.info(f"Found {len(items)} items in folder {folder_id}")
+
+                for item in items:
+                    item_name = item['name']
+                    item_id = item['id']
+                    mime_type = item['mimeType']
+
+                    if mime_type == 'application/vnd.google-apps.folder':
+                        if recursive:
+                            # Create subfolder and recurse
+                            subfolder_path = os.path.join(local_dir, item_name)
+                            os.makedirs(subfolder_path, exist_ok=True)
+                            subfolder_files = self._download_folder_recursive(
+                                item_id,
+                                subfolder_path,
+                                recursive
+                            )
+                            files_downloaded += subfolder_files
+                            logging.info(f"Downloaded {subfolder_files} files from subfolder {item_name}")
+                    else:
+                        # Download file
+                        success = self._download_single_file(item_id, local_dir)
+                        if success:
+                            files_downloaded += 1
+                            logging.info(f"Downloaded file: {item_name}")
+                        else:
+                            logging.warning(f"Failed to download file: {item_name}")
+
+                page_token = results.get('nextPageToken')
+                if not page_token:
+                    break
+
+            return files_downloaded
+
+        except Exception as e:
+            logging.error(f"Error in _download_folder_recursive for folder {folder_id}: {e}", exc_info=True)
+            return files_downloaded
+
+    def _get_extension_for_mime_type(self, mime_type: str) -> str:
+        extensions = {
+            'application/pdf': '.pdf',
+            'text/plain': '.txt',
+            'application/vnd.openxmlformats-officedocument.wordprocessingml.document': '.docx',
+            'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': '.xlsx',
+            'application/vnd.openxmlformats-officedocument.presentationml.presentation': '.pptx',
+            'text/html': '.html',
+            'text/markdown': '.md',
+        }
+        return extensions.get(mime_type, '.bin')
+
+    def _download_folder_contents(self, folder_id: str, local_dir: str, recursive: bool = True) -> int:
+        try:
+            self._ensure_service()
+            return self._download_folder_recursive(folder_id, local_dir, recursive)
+        except Exception as e:
+            logging.error(f"Error downloading folder {folder_id}: {e}", exc_info=True)
+            return 0
+
+    def download_to_directory(self, local_dir: str, source_config: dict = None) -> dict:
+        if source_config is None:
+            source_config = {}
+
+        config = source_config if source_config else getattr(self, 'config', {})
+        files_downloaded = 0
+
+        try:
+            folder_ids = config.get('folder_ids', [])
+            file_ids = config.get('file_ids', [])
+            recursive = config.get('recursive', True)
+
+            self._ensure_service()
+
+            if file_ids:
+                if isinstance(file_ids, str):
+                    file_ids = [file_ids]
+
+                for file_id in file_ids:
+                    if self._download_file_to_directory(file_id, local_dir):
+                        files_downloaded += 1
+
+            # Process folders
+            if folder_ids:
+                if isinstance(folder_ids, str):
+                    folder_ids = [folder_ids]
+
+                for folder_id in folder_ids:
+                    try:
+                        folder_metadata = self.service.files().get(
+                            fileId=folder_id,
+                            fields='name',
+                            supportsAllDrives=True
+                        ).execute()
+                        folder_name = folder_metadata.get('name', '')
+                        folder_path = os.path.join(local_dir, folder_name)
+                        os.makedirs(folder_path, exist_ok=True)
+
+                        folder_files = self._download_folder_recursive(
+                            folder_id,
+                            folder_path,
+                            recursive
+                        )
+                        files_downloaded += folder_files
+                        logging.info(f"Downloaded {folder_files} files from folder {folder_name}")
+                    except Exception as e:
+                        logging.error(f"Error downloading folder {folder_id}: {e}", exc_info=True)
+
+            if not file_ids and not folder_ids:
+                raise ValueError("No folder_ids or file_ids provided for download")
+
+            return {
+                "files_downloaded": files_downloaded,
+                "directory_path": local_dir,
+                "empty_result": files_downloaded == 0,
+                "source_type": "google_drive",
+                "config_used": config
+            }
+
+        except Exception as e:
+            return {
+                "files_downloaded": files_downloaded,
+                "directory_path": local_dir,
+                "empty_result": True,
+                "source_type": "google_drive",
+                "config_used": config,
+                "error": str(e)
+            }

docsgpt/parser/connectors/share_point/__init__.py

@@ -0,0 +1,10 @@
+"""
+Share Point connector package for DocsGPT.
+
+This module provides authentication and document loading capabilities for Share Point.
+"""
+
+from .auth import SharePointAuth
+from .loader import SharePointLoader
+
+__all__ = ['SharePointAuth', 'SharePointLoader']

docsgpt/parser/connectors/share_point/auth.py

@@ -0,0 +1,153 @@
+import datetime
+import logging
+from typing import Optional, Dict, Any
+
+from msal import ConfidentialClientApplication
+
+from docsgpt.core.settings import settings
+from docsgpt.parser.connectors._auth_utils import session_token_fingerprint
+from docsgpt.parser.connectors.base import BaseConnectorAuth
+
+logger = logging.getLogger(__name__)
+
+
+class SharePointAuth(BaseConnectorAuth):
+    """
+    Handles Microsoft OAuth 2.0 authentication for SharePoint/OneDrive.
+
+    Note: Files.Read scope allows access to files the user has granted access to,
+    similar to Google Drive's drive.file scope.
+    """
+
+    SCOPES = [
+        "Files.Read",
+        "Sites.Read.All",
+        "User.Read",
+    ]
+
+    def __init__(self):
+        self.client_id = settings.MICROSOFT_CLIENT_ID
+        self.client_secret = settings.MICROSOFT_CLIENT_SECRET
+
+        if not self.client_id:
+            raise ValueError(
+                "Microsoft OAuth credentials not configured. Please set MICROSOFT_CLIENT_ID in settings."
+            )
+        
+        if not self.client_secret:
+            raise ValueError(
+                "Microsoft OAuth credentials not configured. Please set MICROSOFT_CLIENT_SECRET in settings."
+            )
+
+        self.redirect_uri = settings.CONNECTOR_REDIRECT_BASE_URI
+        self.tenant_id = settings.MICROSOFT_TENANT_ID
+        self.authority = getattr(settings, "MICROSOFT_AUTHORITY", f"https://login.microsoftonline.com/{self.tenant_id}")
+
+        self.auth_app = ConfidentialClientApplication(
+            client_id=self.client_id,
+            client_credential=self.client_secret,
+            authority=self.authority
+        )
+
+    def get_authorization_url(self, state: Optional[str] = None) -> str:
+        return self.auth_app.get_authorization_request_url(
+            scopes=self.SCOPES, state=state, redirect_uri=self.redirect_uri
+        )
+
+    def exchange_code_for_tokens(self, authorization_code: str) -> Dict[str, Any]:
+        result = self.auth_app.acquire_token_by_authorization_code(
+            code=authorization_code,
+            scopes=self.SCOPES,
+            redirect_uri=self.redirect_uri
+        )
+
+        if "error" in result:
+            logger.error("Token exchange failed: %s", result.get("error_description"))
+            raise ValueError(f"Error acquiring token: {result.get('error_description')}")
+
+        return self.map_token_response(result)
+
+    def refresh_access_token(self, refresh_token: str) -> Dict[str, Any]:
+        result = self.auth_app.acquire_token_by_refresh_token(refresh_token=refresh_token, scopes=self.SCOPES)
+
+        if "error" in result:
+            logger.error("Token refresh failed: %s", result.get("error_description"))
+            raise ValueError(f"Error refreshing token: {result.get('error_description')}")
+
+        return self.map_token_response(result)
+
+    def get_token_info_from_session(self, session_token: str) -> Dict[str, Any]:
+        try:
+            from docsgpt.storage.db.repositories.connector_sessions import (
+                ConnectorSessionsRepository,
+            )
+            from docsgpt.storage.db.session import db_readonly
+
+            with db_readonly() as conn:
+                session = ConnectorSessionsRepository(conn).get_by_session_token(
+                    session_token
+                )
+
+            if not session:
+                raise ValueError(
+                    f"Invalid session token ({session_token_fingerprint(session_token)})"
+                )
+
+            token_info = session.get("token_info")
+            if not token_info:
+                raise ValueError("Session missing token information")
+
+            required_fields = ["access_token", "refresh_token"]
+            missing_fields = [field for field in required_fields if field not in token_info or not token_info.get(field)]
+            if missing_fields:
+                raise ValueError(f"Missing required token fields: {missing_fields}")
+
+            if 'token_uri' not in token_info:
+                token_info['token_uri'] = f"https://login.microsoftonline.com/{settings.MICROSOFT_TENANT_ID}/oauth2/v2.0/token"
+
+            return token_info
+
+        except Exception as e:
+            logger.error("Failed to retrieve token from session: %s", e)
+            raise ValueError(f"Failed to retrieve SharePoint token information: {str(e)}")
+
+    def is_token_expired(self, token_info: Dict[str, Any]) -> bool:
+        if not token_info:
+            return True
+
+        expiry_timestamp = token_info.get("expiry")
+
+        if expiry_timestamp is None:
+            return True
+
+        current_timestamp = int(datetime.datetime.now().timestamp())
+        return (expiry_timestamp - current_timestamp) < 60
+
+    def sanitize_token_info(self, token_info: Dict[str, Any], **extra_fields) -> Dict[str, Any]:
+        return super().sanitize_token_info(
+            token_info,
+            allows_shared_content=token_info.get("allows_shared_content", False),
+            **extra_fields,
+        )
+
+    PERSONAL_ACCOUNT_TENANT_ID = "9188040d-6c67-4c5b-b112-36a304b66dad"
+
+    def _allows_shared_content(self, id_token_claims: Dict[str, Any]) -> bool:
+        """Return True when the account is a work/school tenant that can access SharePoint shared content."""
+        tid = id_token_claims.get("tid", "")
+        return bool(tid) and tid != self.PERSONAL_ACCOUNT_TENANT_ID
+
+    def map_token_response(self, result) -> Dict[str, Any]:
+        claims = result.get("id_token_claims", {})
+        return {
+            "access_token": result.get("access_token"),
+            "refresh_token": result.get("refresh_token"),
+            "token_uri": claims.get("iss"),
+            "scopes": result.get("scope"),
+            "expiry": claims.get("exp"),
+            "allows_shared_content": self._allows_shared_content(claims),
+            "user_info": {
+                "name": claims.get("name"),
+                "email": claims.get("preferred_username"),
+            },
+        }

docsgpt/parser/connectors/share_point/loader.py

@@ -0,0 +1,650 @@
+"""
+SharePoint/OneDrive loader for DocsGPT.
+Loads documents from SharePoint/OneDrive using Microsoft Graph API.
+"""
+
+import functools
+import logging
+import os
+from typing import List, Dict, Any, Optional, Tuple
+from urllib.parse import quote
+
+import requests
+
+from docsgpt.parser.connectors.base import BaseConnectorLoader
+from docsgpt.parser.connectors.share_point.auth import SharePointAuth
+from docsgpt.parser.schema.base import Document
+
+
+def _retry_on_auth_failure(func):
+    """Retry once after refreshing the access token on 401/403 responses."""
+    @functools.wraps(func)
+    def wrapper(self, *args, **kwargs):
+        try:
+            return func(self, *args, **kwargs)
+        except requests.exceptions.HTTPError as e:
+            if e.response is not None and e.response.status_code in (401, 403):
+                logging.info(f"Auth failure in {func.__name__}, refreshing token and retrying")
+                try:
+                    new_token_info = self.auth.refresh_access_token(self.refresh_token)
+                    self.access_token = new_token_info.get('access_token')
+                except Exception as refresh_error:
+                    raise ValueError(
+                        f"Authentication failed and could not be refreshed: {refresh_error}"
+                    ) from e
+                return func(self, *args, **kwargs)
+            raise
+    return wrapper
+
+
+class SharePointLoader(BaseConnectorLoader):
+
+    SUPPORTED_MIME_TYPES = {
+        'application/pdf': '.pdf',
+        'application/vnd.openxmlformats-officedocument.wordprocessingml.document': '.docx',
+        'application/vnd.openxmlformats-officedocument.presentationml.presentation': '.pptx',
+        'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': '.xlsx',
+        'application/msword': '.doc',
+        'application/vnd.ms-powerpoint': '.ppt',
+        'application/vnd.ms-excel': '.xls',
+        'text/plain': '.txt',
+        'text/csv': '.csv',
+        'text/html': '.html',
+        'text/markdown': '.md',
+        'text/x-rst': '.rst',
+        'application/json': '.json',
+        'application/epub+zip': '.epub',
+        'application/rtf': '.rtf',
+        'image/jpeg': '.jpg',
+        'image/png': '.png',
+    }
+
+    EXTENSION_TO_MIME = {v: k for k, v in SUPPORTED_MIME_TYPES.items()}
+
+    GRAPH_API_BASE = "https://graph.microsoft.com/v1.0"
+
+    def __init__(self, session_token: str):
+        self.auth = SharePointAuth()
+        self.session_token = session_token
+
+        token_info = self.auth.get_token_info_from_session(session_token)
+        self.access_token = token_info.get('access_token')
+        self.refresh_token = token_info.get('refresh_token')
+        self.allows_shared_content = token_info.get('allows_shared_content', False)
+
+        if not self.access_token:
+            raise ValueError("No access token found in session")
+
+        self.next_page_token = None
+
+    def _get_headers(self) -> Dict[str, str]:
+        return {
+            'Authorization': f'Bearer {self.access_token}',
+            'Accept': 'application/json'
+        }
+
+    def _ensure_valid_token(self):
+        if not self.access_token:
+            raise ValueError("No access token available")
+
+        token_info = {'access_token': self.access_token, 'expiry': None}
+        if self.auth.is_token_expired(token_info):
+            logging.info("Token expired, attempting refresh")
+            try:
+                new_token_info = self.auth.refresh_access_token(self.refresh_token)
+                self.access_token = new_token_info.get('access_token')
+            except Exception:
+                raise ValueError("Failed to refresh access token")
+
+    def _get_item_url(self, item_ref: str) -> str:
+        if ':' in item_ref:
+            drive_id, item_id = item_ref.split(':', 1)
+            return f"{self.GRAPH_API_BASE}/drives/{drive_id}/items/{item_id}"
+        return f"{self.GRAPH_API_BASE}/me/drive/items/{item_ref}"
+
+    def _process_file(self, file_metadata: Dict[str, Any], load_content: bool = True) -> Optional[Document]:
+        try:
+            drive_item_id = file_metadata.get('id')
+            file_name = file_metadata.get('name', 'Unknown')
+            file_data = file_metadata.get('file', {})
+            mime_type = file_data.get('mimeType', 'application/octet-stream')
+
+            if mime_type not in self.SUPPORTED_MIME_TYPES:
+                logging.info(f"Skipping unsupported file type: {mime_type} for file {file_name}")
+                return None
+
+            doc_metadata = {
+                'file_name': file_name,
+                'mime_type': mime_type,
+                'size': file_metadata.get('size'),
+                'created_time': file_metadata.get('createdDateTime'),
+                'modified_time': file_metadata.get('lastModifiedDateTime'),
+                'source': 'share_point'
+            }
+
+            if not load_content:
+                return Document(
+                    text="",
+                    doc_id=drive_item_id,
+                    extra_info=doc_metadata
+                )
+
+            content = self._download_file_content(drive_item_id)
+            if content is None:
+                logging.warning(f"Could not load content for file {file_name} ({drive_item_id})")
+                return None
+
+            return Document(
+                text=content,
+                doc_id=drive_item_id,
+                extra_info=doc_metadata
+            )
+
+        except Exception as e:
+            logging.error(f"Error processing file: {e}")
+            return None
+
+    def load_data(self, inputs: Dict[str, Any]) -> List[Document]:
+        try:
+            documents: List[Document] = []
+
+            folder_id = inputs.get('folder_id')
+            file_ids = inputs.get('file_ids', [])
+            limit = inputs.get('limit', 100)
+            list_only = inputs.get('list_only', False)
+            load_content = not list_only
+            page_token = inputs.get('page_token')
+            search_query = inputs.get('search_query')
+            self.next_page_token = None
+
+            shared = inputs.get('shared', False)
+
+            if file_ids:
+                for file_id in file_ids:
+                    try:
+                        doc = self._load_file_by_id(file_id, load_content=load_content)
+                        if doc:
+                            if not search_query or (
+                                search_query.lower() in doc.extra_info.get('file_name', '').lower()
+                            ):
+                                documents.append(doc)
+                    except Exception as e:
+                        logging.error(f"Error loading file {file_id}: {e}")
+                        continue
+            elif shared:
+                if not self.allows_shared_content:
+                    logging.warning("Shared content is only available for work/school Microsoft accounts")
+                    return []
+                documents = self._list_shared_items(
+                    limit=limit,
+                    load_content=load_content,
+                    page_token=page_token,
+                    search_query=search_query
+                )
+            else:
+                parent_id = folder_id if folder_id else 'root'
+                documents = self._list_items_in_parent(
+                    parent_id,
+                    limit=limit,
+                    load_content=load_content,
+                    page_token=page_token,
+                    search_query=search_query
+                )
+
+            logging.info(f"Loaded {len(documents)} documents from SharePoint/OneDrive")
+            return documents
+
+        except Exception as e:
+            logging.error(f"Error loading data from SharePoint/OneDrive: {e}", exc_info=True)
+            raise
+
+    @_retry_on_auth_failure
+    def _load_file_by_id(self, file_id: str, load_content: bool = True) -> Optional[Document]:
+        self._ensure_valid_token()
+
+        try:
+            url = self._get_item_url(file_id)
+            params = {'$select': 'id,name,file,createdDateTime,lastModifiedDateTime,size'}
+            response = requests.get(url, headers=self._get_headers(), params=params, timeout=100)
+            response.raise_for_status()
+
+            file_metadata = response.json()
+            return self._process_file(file_metadata, load_content=load_content)
+
+        except requests.exceptions.HTTPError:
+            raise
+        except Exception as e:
+            logging.error(f"Error loading file {file_id}: {e}")
+            return None
+
+    @_retry_on_auth_failure
+    def _list_items_in_parent(self, parent_id: str, limit: int = 100, load_content: bool = False, page_token: Optional[str] = None, search_query: Optional[str] = None) -> List[Document]:
+        self._ensure_valid_token()
+
+        documents: List[Document] = []
+
+        try:
+            url = f"{self._get_item_url(parent_id)}/children"
+            params = {'$top': min(100, limit) if limit else 100, '$select': 'id,name,file,folder,createdDateTime,lastModifiedDateTime,size'}
+            if page_token:
+                params['$skipToken'] = page_token
+
+            if search_query:
+                encoded_query = quote(search_query, safe='')
+                if ':' in parent_id:
+                    drive_id = parent_id.split(':', 1)[0]
+                    search_url = f"{self.GRAPH_API_BASE}/drives/{drive_id}/root/search(q='{encoded_query}')"
+                else:
+                    search_url = f"{self.GRAPH_API_BASE}/me/drive/search(q='{encoded_query}')"
+                response = requests.get(search_url, headers=self._get_headers(), params=params, timeout=100)
+            else:
+                response = requests.get(url, headers=self._get_headers(), params=params, timeout=100)
+
+            response.raise_for_status()
+
+            results = response.json()
+
+            items = results.get('value', [])
+            for item in items:
+                if 'folder' in item:
+                    doc_metadata = {
+                        'file_name': item.get('name', 'Unknown'),
+                        'mime_type': 'folder',
+                        'size': item.get('size'),
+                        'created_time': item.get('createdDateTime'),
+                        'modified_time': item.get('lastModifiedDateTime'),
+                        'source': 'share_point',
+                        'is_folder': True
+                    }
+                    documents.append(Document(text="", doc_id=item.get('id'), extra_info=doc_metadata))
+                else:
+                    doc = self._process_file(item, load_content=load_content)
+                    if doc:
+                        documents.append(doc)
+
+                if limit and len(documents) >= limit:
+                    break
+
+            next_link = results.get('@odata.nextLink')
+            if next_link:
+                from urllib.parse import urlparse, parse_qs
+                parsed = urlparse(next_link)
+                query_params = parse_qs(parsed.query)
+                skiptoken_list = query_params.get('$skiptoken')
+                if skiptoken_list:
+                    self.next_page_token = skiptoken_list[0]
+                else:
+                    self.next_page_token = None
+            else:
+                self.next_page_token = None
+            return documents
+
+        except Exception as e:
+            logging.error(f"Error listing items under parent {parent_id}: {e}")
+            return documents
+
+
+
+
+    def _resolve_mime_type(self, resource: Dict[str, Any]) -> Tuple[str, bool]:
+        """Resolve mime type from resource, falling back to file extension."""
+        file_data = resource.get('file', {})
+        mime_type = file_data.get('mimeType') if file_data else None
+
+        if mime_type and mime_type in self.SUPPORTED_MIME_TYPES:
+            return mime_type, True
+
+        name = resource.get('name', '')
+        ext = os.path.splitext(name)[1].lower()
+        if ext in self.EXTENSION_TO_MIME:
+            return self.EXTENSION_TO_MIME[ext], True
+
+        return mime_type or 'application/octet-stream', False
+
+    def _get_user_drive_web_url(self) -> Optional[str]:
+        """Fetch the current user's OneDrive web URL for KQL path exclusion."""
+        try:
+            response = requests.get(
+                f"{self.GRAPH_API_BASE}/me/drive",
+                headers=self._get_headers(),
+                params={'$select': 'webUrl'},
+                timeout=100,
+            )
+            response.raise_for_status()
+            return response.json().get('webUrl')
+        except Exception as e:
+            logging.warning(f"Could not fetch user drive web URL: {e}")
+            return None
+
+    def _build_shared_kql_query(self, search_query: Optional[str], user_drive_url: Optional[str]) -> str:
+        """Build KQL query string that excludes the user's own drive items."""
+        base_query = search_query if search_query else "*"
+        if user_drive_url:
+            return f'{base_query} AND -path:"{user_drive_url}"'
+        return base_query
+
+    def _list_shared_items(self, limit: int = 100, load_content: bool = False, page_token: Optional[str] = None, search_query: Optional[str] = None) -> List[Document]:
+        """Fetch shared drive items using Microsoft Graph Search API with local offset paging.
+
+        We always fetch up to a fixed maximum number of hits from Graph (single request),
+        then page through that array locally using `page_token` as a simple integer offset.
+        This avoids relying on buggy or inconsistent remote `from`/`size` semantics.
+        """
+        self._ensure_valid_token()
+        documents: List[Document] = []
+
+        try:
+            user_drive_url = self._get_user_drive_web_url()
+            query_text = self._build_shared_kql_query(search_query, user_drive_url)
+
+            url = f"{self.GRAPH_API_BASE}/search/query"
+            page_size = 500  # maximum number of hits we care about for selection
+
+            body = {
+                "requests": [
+                    {
+                        "entityTypes": ["driveItem"],
+                        "query": {"queryString": query_text},
+                        "from": 0,
+                        "size": page_size,
+                    }
+                ]
+            }
+
+            headers = self._get_headers()
+            headers["Content-Type"] = "application/json"
+            response = requests.post(url, headers=headers, json=body, timeout=100)
+            response.raise_for_status()
+            results = response.json()
+
+            search_response = results.get("value", [])
+            if not search_response:
+                logging.warning("Search API returned empty value array")
+                self.next_page_token = None
+                return documents
+
+            hits_containers = search_response[0].get("hitsContainers", [])
+            if not hits_containers:
+                logging.warning("Search API returned no hitsContainers")
+                self.next_page_token = None
+                return documents
+
+            container = hits_containers[0]
+            total = container.get("total", 0)
+            raw_hits = container.get("hits", [])
+
+            # Deduplicate by effective item ID (driveId:itemId) to avoid the same
+            # resource appearing multiple times across the result set.
+            deduped_hits = []
+            seen_ids = set()
+            for hit in raw_hits:
+                resource = hit.get("resource", {})
+                item_id = resource.get("id")
+                drive_id = resource.get("parentReference", {}).get("driveId")
+                effective_id = f"{drive_id}:{item_id}" if drive_id and item_id else item_id
+                if not effective_id or effective_id in seen_ids:
+                    continue
+                seen_ids.add(effective_id)
+                deduped_hits.append(hit)
+
+            hits = deduped_hits
+            logging.info(
+                f"Search API returned {total} total results, {len(raw_hits)} raw hits, {len(hits)} unique hits in this batch"
+            )
+            try:
+                offset = int(page_token) if page_token is not None else 0
+            except (TypeError, ValueError):
+                logging.warning(
+                    f"Invalid page_token '{page_token}' for shared items search, defaulting to 0"
+                )
+                offset = 0
+
+            if offset < 0:
+                offset = 0
+            if offset >= len(hits):
+                self.next_page_token = None
+                return documents
+
+            end_index = offset + limit if limit else len(hits)
+            end_index = min(end_index, len(hits))
+
+            for hit in hits[offset:end_index]:
+                resource = hit.get("resource", {})
+                item_name = resource.get("name", "Unknown")
+                item_id = resource.get("id")
+                drive_id = resource.get("parentReference", {}).get("driveId")
+
+                effective_id = f"{drive_id}:{item_id}" if drive_id and item_id else item_id
+
+                is_folder = "folder" in resource
+
+                if is_folder:
+                    doc_metadata = {
+                        "file_name": item_name,
+                        "mime_type": "folder",
+                        "size": resource.get("size"),
+                        "created_time": resource.get("createdDateTime"),
+                        "modified_time": resource.get("lastModifiedDateTime"),
+                        "source": "share_point",
+                        "is_folder": True,
+                    }
+                    documents.append(
+                        Document(text="", doc_id=effective_id, extra_info=doc_metadata)
+                    )
+                else:
+                    mime_type, supported = self._resolve_mime_type(resource)
+                    if not supported:
+                        logging.info(
+                            f"Skipping unsupported shared file: {item_name} (mime: {mime_type})"
+                        )
+                        continue
+
+                    doc_metadata = {
+                        "file_name": item_name,
+                        "mime_type": mime_type,
+                        "size": resource.get("size"),
+                        "created_time": resource.get("createdDateTime"),
+                        "modified_time": resource.get("lastModifiedDateTime"),
+                        "source": "share_point",
+                    }
+
+                    content = ""
+                    if load_content:
+                        content = self._download_file_content(effective_id) or ""
+
+                    documents.append(
+                        Document(text=content, doc_id=effective_id, extra_info=doc_metadata)
+                    )
+
+            if limit and end_index < len(hits):
+                self.next_page_token = str(end_index)
+            else:
+                self.next_page_token = None
+
+            return documents
+
+        except Exception as e:
+            logging.error(f"Error listing shared items via search API: {e}", exc_info=True)
+            return documents
+
+    @_retry_on_auth_failure
+    def _download_file_content(self, file_id: str) -> Optional[str]:
+        self._ensure_valid_token()
+
+        try:
+            url = f"{self._get_item_url(file_id)}/content"
+            response = requests.get(url, headers=self._get_headers(), timeout=100)
+            response.raise_for_status()
+
+            try:
+                return response.content.decode('utf-8')
+            except UnicodeDecodeError:
+                logging.error(f"Could not decode file {file_id} as text")
+                return None
+
+        except requests.exceptions.HTTPError:
+            raise
+        except Exception as e:
+            logging.error(f"Error downloading file {file_id}: {e}")
+            return None
+
+    def _download_single_file(self, file_id: str, local_dir: str) -> bool:
+        try:
+            url = self._get_item_url(file_id)
+            params = {'$select': 'id,name,file'}
+            response = requests.get(url, headers=self._get_headers(), params=params, timeout=100)
+            response.raise_for_status()
+
+            metadata = response.json()
+            file_name = metadata.get('name', 'unknown')
+            file_data = metadata.get('file', {})
+            mime_type = file_data.get('mimeType', 'application/octet-stream')
+
+            if mime_type not in self.SUPPORTED_MIME_TYPES:
+                logging.info(f"Skipping unsupported file type: {mime_type}")
+                return False
+
+            os.makedirs(local_dir, exist_ok=True)
+            full_path = os.path.join(local_dir, file_name)
+
+            download_url = f"{self._get_item_url(file_id)}/content"
+            download_response = requests.get(download_url, headers=self._get_headers(), timeout=100)
+            download_response.raise_for_status()
+
+            with open(full_path, 'wb') as f:
+                f.write(download_response.content)
+
+            return True
+        except Exception as e:
+            logging.error(f"Error in _download_single_file: {e}")
+            return False
+
+    def _download_folder_recursive(self, folder_id: str, local_dir: str, recursive: bool = True) -> int:
+        files_downloaded = 0
+        try:
+            os.makedirs(local_dir, exist_ok=True)
+
+            url = f"{self._get_item_url(folder_id)}/children"
+            params = {'$top': 1000}
+
+            while url:
+                response = requests.get(url, headers=self._get_headers(), params=params, timeout=100)
+                response.raise_for_status()
+
+                results = response.json()
+                items = results.get('value', [])
+                logging.info(f"Found {len(items)} items in folder {folder_id}")
+
+                for item in items:
+                    item_name = item.get('name', 'unknown')
+                    item_id = item.get('id')
+
+                    if 'folder' in item:
+                        if recursive:
+                            subfolder_path = os.path.join(local_dir, item_name)
+                            os.makedirs(subfolder_path, exist_ok=True)
+                            subfolder_files = self._download_folder_recursive(
+                                item_id,
+                                subfolder_path,
+                                recursive
+                            )
+                            files_downloaded += subfolder_files
+                            logging.info(f"Downloaded {subfolder_files} files from subfolder {item_name}")
+                    else:
+                        success = self._download_single_file(item_id, local_dir)
+                        if success:
+                            files_downloaded += 1
+                            logging.info(f"Downloaded file: {item_name}")
+                        else:
+                            logging.warning(f"Failed to download file: {item_name}")
+
+                url = results.get('@odata.nextLink')
+
+            return files_downloaded
+
+        except Exception as e:
+            logging.error(f"Error in _download_folder_recursive for folder {folder_id}: {e}", exc_info=True)
+            return files_downloaded
+
+    def _download_folder_contents(self, folder_id: str, local_dir: str, recursive: bool = True) -> int:
+        try:
+            self._ensure_valid_token()
+            return self._download_folder_recursive(folder_id, local_dir, recursive)
+        except Exception as e:
+            logging.error(f"Error downloading folder {folder_id}: {e}", exc_info=True)
+            return 0
+
+    def _download_file_to_directory(self, file_id: str, local_dir: str) -> bool:
+        try:
+            self._ensure_valid_token()
+            return self._download_single_file(file_id, local_dir)
+        except Exception as e:
+            logging.error(f"Error downloading file {file_id}: {e}", exc_info=True)
+            return False
+
+    def download_to_directory(self, local_dir: str, source_config: Dict[str, Any] = None) -> Dict[str, Any]:
+        if source_config is None:
+            source_config = {}
+
+        config = source_config if source_config else getattr(self, 'config', {})
+        files_downloaded = 0
+
+        try:
+            folder_ids = config.get('folder_ids', [])
+            file_ids = config.get('file_ids', [])
+            recursive = config.get('recursive', True)
+
+            if file_ids:
+                if isinstance(file_ids, str):
+                    file_ids = [file_ids]
+
+                for file_id in file_ids:
+                    if self._download_file_to_directory(file_id, local_dir):
+                        files_downloaded += 1
+
+            if folder_ids:
+                if isinstance(folder_ids, str):
+                    folder_ids = [folder_ids]
+
+                for folder_id in folder_ids:
+                    try:
+                        url = self._get_item_url(folder_id)
+                        params = {'$select': 'id,name'}
+                        response = requests.get(url, headers=self._get_headers(), params=params, timeout=100)
+                        response.raise_for_status()
+
+                        folder_metadata = response.json()
+                        folder_name = folder_metadata.get('name', '')
+                        folder_path = os.path.join(local_dir, folder_name)
+                        os.makedirs(folder_path, exist_ok=True)
+
+                        folder_files = self._download_folder_recursive(
+                            folder_id,
+                            folder_path,
+                            recursive
+                        )
+                        files_downloaded += folder_files
+                        logging.info(f"Downloaded {folder_files} files from folder {folder_name}")
+                    except Exception as e:
+                        logging.error(f"Error downloading folder {folder_id}: {e}", exc_info=True)
+
+            if not file_ids and not folder_ids:
+                raise ValueError("No folder_ids or file_ids provided for download")
+
+            return {
+                "files_downloaded": files_downloaded,
+                "directory_path": local_dir,
+                "empty_result": files_downloaded == 0,
+                "source_type": "share_point",
+                "config_used": config
+            }
+
+        except Exception as e:
+            return {
+                "files_downloaded": files_downloaded,
+                "directory_path": local_dir,
+                "empty_result": True,
+                "source_type": "share_point",
+                "config_used": config,
+                "error": str(e)
+            }

docsgpt/parser/document_reader.py

@@ -0,0 +1,628 @@
+"""In-process document parsing for the ``read_document`` tool, run on the Celery parsing worker.
+
+``parse_document_bytes`` turns untrusted document bytes into a bounded, shaped
+result (markdown/text/structured/chunks) using the BACKEND parsers (the engine
+selected by ``DOC_PARSER_ENGINE``; ``structured`` output always needs Docling).
+It applies the same untrusted-content safeguards as uploads — an extension
+whitelist, a byte cap, ``safe_filename`` staging into a temp file, and temp
+cleanup — so a hostile filename or document is treated as inert data.
+"""
+
+from __future__ import annotations
+
+import io
+import logging
+import os
+import tempfile
+import zipfile
+from pathlib import Path
+from typing import Any, Dict, Iterator, List, Optional, Union
+
+from docsgpt.core.settings import settings
+from docsgpt.parser.file.bulk import get_default_file_extractor
+from docsgpt.parser.file.constants import SUPPORTED_SOURCE_EXTENSIONS
+from docsgpt.utils import safe_filename
+
+logger = logging.getLogger(__name__)
+
+# Default cap for the LLM-facing VIEW of the text (applied in ``bound_parse_payload``,
+# NOT during parsing) so a huge document can't flood context; the full result is still
+# persisted as a ``data`` artifact. When the text exceeds the cap a head+tail window keeps
+# both the document's beginning AND end (e.g. totals/signatures) within the byte budget.
+_TEXT_MAX_BYTES = 8000
+_MAX_TABLES_RETURNED = 20
+_MAX_TABLE_ROWS = 50
+_MAX_CELL_CHARS = 200
+# Caps applied to the bounded view that rides back through the Redis result
+# backend (the full result still lives in the persisted artifact).
+_MAX_CHUNKS_RETURNED = 50
+_MAX_PAGE_SELECTIONS = 10_000
+_MAX_PAGE_SELECTOR_TOKENS = 20_000
+
+_VALID_OUTPUTS = ("markdown", "text", "structured", "chunks")
+_VALID_OCR = ("auto", "on", "off")
+_VALID_ENGINES = ("auto", "docling", "anydoc", "fast")
+
+
+def truncate_text_head_tail(text: str, max_bytes: Optional[int] = None) -> str:
+    """Bound text to a head+tail byte window so a large file can't flood context."""
+    cap = int(max_bytes or _TEXT_MAX_BYTES)
+    if cap <= 0:
+        return text
+    encoded = text.encode("utf-8")
+    if len(encoded) <= cap:
+        return text
+    head = cap // 2
+    tail = cap - head
+    dropped = len(encoded) - head - tail
+    head_text = encoded[:head].decode("utf-8", errors="ignore")
+    tail_text = encoded[-tail:].decode("utf-8", errors="ignore")
+    return f"{head_text}\n\n...[truncated {dropped} bytes]...\n\n{tail_text}"
+
+
+def bound_parse_payload(payload: Dict[str, Any], max_chars: Optional[int] = None) -> Dict[str, Any]:
+    """Bound every shape of a parse payload so the Redis-backed result stays small.
+
+    This is where ALL view-bounding happens: parsing now returns the FULL content and the
+    persisted ``data`` artifact keeps it, while the view ridden back through the Redis result
+    backend is bounded here. ``content`` is capped to ``max_chars`` when given, else re-windowed
+    to a head+tail byte window; ``chunks`` is capped in count and per-chunk length. ``structured``
+    is left as-is: it rides back so json_schema validation in the tool can run against it, and it
+    is already bounded by the input byte cap plus the table caps (``_compact_table`` /
+    ``summary``). ``payload['truncated']`` is set when the content view actually cut. The dict is
+    mutated in place.
+    """
+    content = payload.get("content")
+    if isinstance(content, str):
+        if max_chars and int(max_chars) > 0:
+            capped = content[: int(max_chars)]
+        else:
+            capped = truncate_text_head_tail(content)
+        payload["content"] = capped
+        payload["truncated"] = capped != content
+
+    chunks = payload.get("chunks")
+    if isinstance(chunks, list):
+        bounded = [
+            truncate_text_head_tail(chunk) if isinstance(chunk, str) else chunk
+            for chunk in chunks[:_MAX_CHUNKS_RETURNED]
+        ]
+        if len(chunks) > _MAX_CHUNKS_RETURNED:
+            payload["chunks_truncated"] = True
+            payload["total_chunks"] = len(chunks)
+        payload["chunks"] = bounded
+
+    return payload
+
+
+def _max_input_bytes() -> int:
+    """Return the size cap for a parsed document (its own setting, else the sandbox cap)."""
+    explicit = int(getattr(settings, "DOCUMENT_PARSE_MAX_BYTES", 0) or 0)
+    if explicit > 0:
+        return explicit
+    return int(getattr(settings, "SANDBOX_MAX_INPUT_BYTES", 25 * 1024 * 1024))
+
+
+# Every zip-packaged format a parser map can route: OOXML and its macro/
+# binary variants, OpenDocument, EPUB. Chat attachments have no extension
+# whitelist, so the anydoc engine can receive any of these.
+_ZIP_CONTAINER_EXTENSIONS = frozenset({
+    ".docx", ".docm", ".xlsx", ".xlsm", ".xlsb", ".pptx", ".pptm", ".ppsx", ".ppsm",
+    ".odt", ".ods", ".odp", ".epub",
+})
+
+
+def _zip_bomb_reason(source: Union[bytes, str, Path], suffix: str) -> Optional[str]:
+    """Return an error string if a zip container declares an implausible expansion, else None.
+
+    Reads only the central directory (declared sizes), never decompressing.
+
+    Args:
+        source: Document bytes, or a filesystem path to the document.
+        suffix: Lowercased file extension; non-zip formats are not gated here.
+
+    Returns:
+        The rejection reason, or None when the archive is acceptable, is not a
+        zip container, or is not a readable zip.
+    """
+    if suffix not in _ZIP_CONTAINER_EXTENSIONS:
+        return None
+    max_entries = int(getattr(settings, "DOCUMENT_MAX_ARCHIVE_ENTRIES", 10000))
+    cap = int(getattr(settings, "DOCUMENT_MAX_DECOMPRESSED_BYTES", 300 * 1024 * 1024))
+    opened = io.BytesIO(source) if isinstance(source, bytes) else source
+    try:
+        with zipfile.ZipFile(opened) as zf:
+            infos = zf.infolist()
+            if len(infos) > max_entries:
+                return f"document archive has too many entries ({len(infos)} > {max_entries})."
+            total = 0
+            for info in infos:
+                total += info.file_size
+                if total > cap:
+                    return f"document decompresses to too much data: exceeds the {cap}-byte cap."
+    except zipfile.BadZipFile:
+        # Not a readable zip; the format-specific parser will surface a clean error.
+        return None
+    return None
+
+
+def _reject_zip_bomb(data: bytes, suffix: str) -> Optional[str]:
+    """Return an error string if a zip-based document declares an implausible expansion, else None."""
+    return _zip_bomb_reason(data, suffix)
+
+
+def reject_zip_bomb_path(path: Union[str, Path]) -> Optional[str]:
+    """Path-taking sibling of ``_reject_zip_bomb`` for callers that have a file, not bytes.
+
+    Args:
+        path: Filesystem path of the document about to be parsed.
+
+    Returns:
+        The rejection reason, or None when the file is acceptable / not a zip container.
+    """
+    return _zip_bomb_reason(path, os.path.splitext(str(path))[1].lower())
+
+
+def _resolve_ocr_enabled(ocr: str) -> bool:
+    """Resolve the OCR flag from the ``ocr`` arg and the deployment setting."""
+    if ocr == "on":
+        return True
+    if ocr == "off":
+        return False
+    return bool(getattr(settings, "OCR_ENABLED", False))
+
+
+def _effective_engine(engine: str) -> str:
+    """Resolve ``auto`` to the server's ``DOC_PARSER_ENGINE``; other values pass through."""
+    if engine != "auto":
+        return engine
+    configured = getattr(settings, "DOC_PARSER_ENGINE", None) or "anydoc"
+    return str(configured).strip().lower()
+
+
+def _pick_parser(suffix: str, *, ocr_enabled: bool, engine: str):
+    """Select the parser for ``suffix`` honoring the requested engine; None when unsupported."""
+    if engine == "fast":
+        legacy = _legacy_parser_for(suffix)
+        if legacy is not None:
+            return legacy
+        from docsgpt.parser.file.anydoc_parser import ANYDOC_GAINED_SUFFIXES
+
+        if suffix in ANYDOC_GAINED_SUFFIXES:
+            # anydoc-only office formats have no legacy parser. Falling through
+            # would run the configured engine the caller explicitly opted out
+            # of; images/audio/.txt still fall through (no legacy alternative).
+            return None
+    if engine in ("docling", "anydoc"):
+        extractor = get_default_file_extractor(ocr_enabled=ocr_enabled, engine=engine)
+    else:
+        extractor = get_default_file_extractor(ocr_enabled=ocr_enabled)
+    return extractor.get(suffix)
+
+
+def _legacy_parser_for(suffix: str):
+    """Return a non-Docling parser for ``suffix`` (the ``fast`` engine), or None."""
+    from docsgpt.parser.file.docs_parser import DocxParser, PDFParser
+    from docsgpt.parser.file.epub_parser import EpubParser
+    from docsgpt.parser.file.html_parser import HTMLParser
+    from docsgpt.parser.file.json_parser import JSONParser
+    from docsgpt.parser.file.markdown_parser import MarkdownParser
+    from docsgpt.parser.file.pptx_parser import PPTXParser
+    from docsgpt.parser.file.rst_parser import RstParser
+    from docsgpt.parser.file.tabular_parser import ExcelParser, PandasCSVParser
+
+    legacy = {
+        ".pdf": PDFParser,
+        ".docx": DocxParser,
+        ".pptx": PPTXParser,
+        ".csv": PandasCSVParser,
+        ".xlsx": ExcelParser,
+        ".html": HTMLParser,
+        ".xhtml": HTMLParser,
+        ".epub": EpubParser,
+        ".md": MarkdownParser,
+        ".mdx": MarkdownParser,
+        ".rst": RstParser,
+        ".json": JSONParser,
+    }
+    cls = legacy.get(suffix)
+    return cls() if cls is not None else None
+
+
+def _parse_to_text(parser: Any, path: Path) -> str:
+    """Run a parser and coerce its ``str | List[str]`` result to a single text blob."""
+    if not parser.parser_config_set:
+        parser.init_parser()
+    parsed = parser.parse_file(path, errors="ignore")
+    if isinstance(parsed, list):
+        return "\n\n".join(str(part) for part in parsed)
+    return str(parsed)
+
+
+def _is_native_ocr_parser(parser: Any) -> bool:
+    """True for the native OCR PDF parser (OCR_BACKEND=native): a docling table pass would OCR the scan again."""
+    if parser is None:
+        return False
+    try:
+        from docsgpt.parser.file.ocr_parser import NativeOcrPdfParser
+    except Exception:
+        return False
+    return isinstance(parser, NativeOcrPdfParser)
+
+
+def _is_docling_parser(parser: Any) -> bool:
+    """True when ``parser`` is Docling-backed (collecting tables would otherwise re-convert)."""
+    if parser is None:
+        return False
+    try:
+        from docsgpt.parser.file.docling_parser import DoclingParser
+    except Exception:
+        return False
+    return isinstance(parser, DoclingParser)
+
+
+def _compact_table(table: Dict[str, Any]) -> Dict[str, Any]:
+    """Bound a single table's rows and cell sizes so one giant table can't bloat context."""
+
+    def _cell(value: Any) -> Any:
+        if isinstance(value, str) and len(value) > _MAX_CELL_CHARS:
+            return value[:_MAX_CELL_CHARS] + "...[truncated]"
+        return value
+
+    rows = table.get("rows")
+    if not isinstance(rows, list):
+        return table
+    capped = [[_cell(c) for c in row] if isinstance(row, list) else _cell(row) for row in rows[:_MAX_TABLE_ROWS]]
+    compact = dict(table)
+    compact["rows"] = capped
+    if len(rows) > _MAX_TABLE_ROWS:
+        compact["rows_truncated"] = True
+        compact["total_rows"] = len(rows)
+    return compact
+
+
+def _docling_structured(path: Path, *, ocr_enabled: bool, include_tables: bool, parser: Any = None) -> Dict[str, Any]:
+    """Convert a document with Docling and return markdown + structured dict + bounded tables.
+
+    When ``parser`` is the configured ``DoclingParser`` (the collapse-the-double-convert
+    path), reuse ITS converter + export so OCR/pipeline options and the export fallback
+    are honored and the content matches the legacy single-parse output exactly; otherwise
+    fall back to a vanilla converter.
+    """
+    if _is_docling_parser(parser):
+        if getattr(parser, "_converter", None) is None:
+            parser._init_parser()
+        doc = parser._converter.convert(str(path)).document
+        markdown = parser._export_content(doc)
+    else:
+        from docling.document_converter import DocumentConverter
+
+        from docsgpt.parser.file.docling_parser import _apply_inference_settings
+
+        _apply_inference_settings()
+        converter = DocumentConverter()
+        doc = converter.convert(str(path)).document
+        markdown = doc.export_to_markdown()
+    structured = doc.export_to_dict()
+    tables: List[Dict[str, Any]] = []
+    if include_tables:
+        for tbl in getattr(doc, "tables", []) or []:
+            try:
+                from docsgpt.parser.file.tabular_parser import cell_to_text
+
+                # ``astype(str)`` leaks pandas-3 NaNs and mangles gapped int
+                # columns ("1001.0") — same contract as the tabular parser.
+                df = tbl.export_to_dataframe()
+                tables.append({
+                    "columns": [cell_to_text(c) for c in df.columns],
+                    "rows": [[cell_to_text(v) for v in row] for row in df.values.tolist()],
+                })
+            except Exception:
+                try:
+                    tables.append({"markdown": tbl.export_to_markdown()})
+                except Exception:
+                    continue
+            if len(tables) >= _MAX_TABLES_RETURNED:
+                break
+    page_count = len(getattr(doc, "pages", {}) or {})
+    return {"markdown": markdown, "structured": structured, "tables": tables, "page_count": page_count}
+
+
+def _structure_summary(structured: Any) -> Dict[str, int]:
+    """Summarize the Docling structured dict by top-level element counts (keeps context compact)."""
+    if not isinstance(structured, dict):
+        return {}
+    counts: Dict[str, int] = {}
+    for key in ("texts", "tables", "pictures", "groups", "pages"):
+        value = structured.get(key)
+        if isinstance(value, (list, dict)):
+            counts[key] = len(value)
+    return counts
+
+
+def _apply_pages(text: str, pages: Any) -> str:
+    """Best-effort page-range slice on a page-delimited markdown blob (``\\f`` separated)."""
+    if not pages or "\f" not in text:
+        return text
+    total = text.count("\f") + 1
+    selected = _selected_page_indices(pages, total)
+    if not selected:
+        return text
+
+    # Splitting a hostile form-feed blob could allocate millions of strings.
+    # Walk boundaries and retain only the bounded set of requested pages.
+    wanted = set(selected)
+    found: Dict[int, tuple[int, int]] = {}
+    page_index = 0
+    start = 0
+    while wanted:
+        end = text.find("\f", start)
+        if end < 0:
+            end = len(text)
+        if page_index in wanted:
+            found[page_index] = (start, end)
+            wanted.remove(page_index)
+        if end == len(text):
+            break
+        page_index += 1
+        start = end + 1
+
+    # Preserve requested order and ordinary duplicates, but never let repeated
+    # selectors amplify the output beyond the source text's resident size.
+    output = io.StringIO()
+    output_chars = 0
+    wrote_page = False
+    for index in selected:
+        span = found.get(index)
+        if span is None:
+            continue
+        page_start, page_end = span
+        added = (page_end - page_start) + (1 if wrote_page else 0)
+        if output_chars + added > len(text):
+            break
+        if wrote_page:
+            output.write("\f")
+        output.write(text[page_start:page_end])
+        output_chars += added
+        wrote_page = True
+    return output.getvalue() if wrote_page else text
+
+
+def _iter_page_tokens(pages: Any) -> Iterator[Any]:
+    """Yield bounded selector tokens without materializing a comma-split list."""
+    if isinstance(pages, list):
+        for position, token in enumerate(pages):
+            if position >= _MAX_PAGE_SELECTOR_TOKENS:
+                break
+            yield token
+        return
+
+    raw = str(pages)
+    start = 0
+    emitted = 0
+    while emitted < _MAX_PAGE_SELECTOR_TOKENS:
+        end = raw.find(",", start)
+        if end < 0:
+            yield raw[start:]
+            return
+        yield raw[start:end]
+        emitted += 1
+        start = end + 1
+
+
+def _selected_page_indices(pages: Any, total: int) -> List[int]:
+    """Parse ``pages`` into a bounded list of valid 0-based page occurrences."""
+    if total <= 0:
+        return []
+
+    indices: List[int] = []
+    for token in _iter_page_tokens(pages):
+        if len(indices) >= _MAX_PAGE_SELECTIONS:
+            break
+        token = str(token).strip()
+        if "-" in token:
+            try:
+                lo, hi = (int(p) for p in token.split("-", 1))
+            except ValueError:
+                continue
+            start = max(lo - 1, 0)
+            stop = min(hi, total, start + (_MAX_PAGE_SELECTIONS - len(indices)))
+            indices.extend(range(start, stop))
+        else:
+            try:
+                index = int(token) - 1
+            except ValueError:
+                continue
+            if 0 <= index < total:
+                indices.append(index)
+    return indices
+
+
+def _to_chunks(text: str, max_chars: Optional[int]) -> List[str]:
+    """Chunk parsed text via the ingestion chunker; bounded and JSON-safe for the result."""
+    from docsgpt.parser.chunking_creator import ChunkerCreator
+    from docsgpt.parser.schema.base import Document
+
+    chunker = ChunkerCreator.create_chunker("classic_chunk")
+    chunks = chunker.chunk([Document(text=text)])
+    cap = int(max_chars or 0)
+    out: List[str] = []
+    for chunk in chunks:
+        body = getattr(chunk, "text", str(chunk))
+        out.append(body[:cap] if cap > 0 else body)
+        if len(out) >= 200:
+            break
+    return out
+
+
+def parse_document_bytes(
+    data: bytes,
+    filename: str,
+    *,
+    output: str = "markdown",
+    ocr: str = "auto",
+    pages: Any = None,
+    engine: str = "auto",
+    max_chars: Optional[int] = None,
+    include_tables: bool = True,
+) -> Dict[str, Any]:
+    """Parse untrusted document bytes into a bounded shaped result; whitelist + size + cleanup guarded."""
+    if output not in _VALID_OUTPUTS:
+        return {"error": f"unsupported output '{output}'; expected one of {_VALID_OUTPUTS}."}
+    if ocr not in _VALID_OCR:
+        return {"error": f"unsupported ocr '{ocr}'; expected one of {_VALID_OCR}."}
+    if engine not in _VALID_ENGINES:
+        return {"error": f"unsupported engine '{engine}'; expected one of {_VALID_ENGINES}."}
+
+    safe_name = safe_filename(filename) or "document"
+    suffix = os.path.splitext(safe_name)[1].lower()
+    if suffix not in SUPPORTED_SOURCE_EXTENSIONS:
+        return {"error": f"unsupported file type '{suffix or filename}'."}
+
+    cap = _max_input_bytes()
+    if len(data) > cap:
+        return {"error": f"input document is too large: {len(data)} bytes exceeds the {cap}-byte cap."}
+
+    bomb = _reject_zip_bomb(data, suffix)
+    if bomb is not None:
+        return {"error": bomb}
+
+    ocr_enabled = _resolve_ocr_enabled(ocr)
+    tmp_dir = tempfile.mkdtemp(prefix="docparse-")
+    tmp_path = Path(tmp_dir) / safe_name
+    try:
+        tmp_path.write_bytes(data)
+        return _shape(tmp_path, suffix, output, ocr_enabled, engine, pages, max_chars, include_tables)
+    except Exception as exc:
+        logger.exception("parse_document_bytes: parsing failed")
+        return {"error": f"parsing failed: {type(exc).__name__}: {exc}"}
+    finally:
+        try:
+            tmp_path.unlink(missing_ok=True)
+            os.rmdir(tmp_dir)
+        except OSError:
+            logger.warning("parse_document_bytes: temp cleanup failed for %s", tmp_dir, exc_info=True)
+
+
+def _shape(
+    path: Path,
+    suffix: str,
+    output: str,
+    ocr_enabled: bool,
+    engine: str,
+    pages: Any,
+    max_chars: Optional[int],
+    include_tables: bool,
+) -> Dict[str, Any]:
+    """Run the selected parser/engine and shape the result per ``output``; bounded throughout.
+
+    ``output='structured'`` always uses Docling regardless of ``engine`` — the ``fast``
+    engine is markdown/text only and cannot produce the structured dict.
+    """
+    if output == "structured":
+        try:
+            extracted = _docling_structured(path, ocr_enabled=ocr_enabled, include_tables=include_tables)
+        except Exception as exc:
+            return {"error": f"structured parsing requires Docling: {type(exc).__name__}: {exc}"}
+        bounded, truncated = _bounded(extracted["markdown"])
+        return {
+            "output": "structured",
+            "content": bounded,
+            "truncated": truncated,
+            "tables": [_compact_table(t) for t in extracted["tables"]],
+            "structured": extracted["structured"],
+            "summary": _structure_summary(extracted["structured"]),
+            "page_count": extracted["page_count"],
+        }
+
+    parser = _pick_parser(suffix, ocr_enabled=ocr_enabled, engine=engine)
+    # Under OCR_BACKEND=native with the docling engine the PDF parser is the
+    # native OCR parser wrapping a DoclingParser as its text_parser. A PDF
+    # with a text layer on every page is handed to that parser untouched, so
+    # when tables are wanted run it directly: the single docling conversion
+    # below then yields both the markdown and the tables. Scans and mixed
+    # documents keep the native parser (and the exclusion just below).
+    if include_tables and output != "chunks" and _is_native_ocr_parser(parser):
+        delegate = parser.text_layer_delegate(path)
+        if _is_docling_parser(delegate):
+            parser = delegate
+    # Tables come from a Docling conversion, so they are only collected when
+    # a Docling parser is actually in play: under anydoc/fast a table pass
+    # would be a second, full docling conversion that costs more than the
+    # whole read, and under OCR_BACKEND=native the PDF parser is the native
+    # OCR parser (docling only as its text_parser), where a vanilla
+    # DocumentConverter pass would OCR the scan a second time with docling's
+    # default engine, ignoring OCR_ENGINE.
+    wants_tables = (
+        include_tables
+        and output != "chunks"
+        and _effective_engine(engine) == "docling"
+        and not _is_native_ocr_parser(parser)
+    )
+
+    # A Docling-backed parser already converts the whole document to produce its text.
+    # When tables are also requested, reuse that single conversion for both the markdown
+    # content and the tables instead of converting a second time just to collect tables
+    # (Docling/torch conversion dominates the cost, so a re-convert ~doubles latency).
+    if wants_tables and _is_docling_parser(parser):
+        try:
+            extracted = _docling_structured(path, ocr_enabled=ocr_enabled, include_tables=True, parser=parser)
+            text = extracted["markdown"]
+            tables: List[Dict[str, Any]] = [_compact_table(t) for t in extracted["tables"]]
+        except Exception:
+            text, tables = _parse_to_text(parser, path), []
+        text = _apply_pages(text, pages)
+        bounded, truncated = _bounded(text)
+        payload: Dict[str, Any] = {"output": output, "content": bounded, "truncated": truncated}
+        if tables:
+            payload["tables"] = tables
+        return payload
+
+    if parser is None:
+        # A whitelisted extension with no dedicated parser (e.g. .txt) reads as plain
+        # text, matching SimpleDirectoryReader's standard-read fallback. Binary
+        # office formats that only anydoc reads must not: without anydoc they
+        # would come back as OLE/zip bytes decoded as text.
+        from docsgpt.parser.file.anydoc_parser import ANYDOC_GAINED_SUFFIXES
+
+        if suffix in ANYDOC_GAINED_SUFFIXES:
+            if engine == "fast":
+                return {
+                    "error": f"The fast engine has no parser for {suffix} files; "
+                    "use engine='anydoc' or 'auto'."
+                }
+            return {
+                "error": f"No parser is available for {suffix} files: firecrawl-anydoc "
+                "is not installed (pip install firecrawl-anydoc)"
+            }
+        text = path.read_text(errors="ignore")
+    else:
+        text = _parse_to_text(parser, path)
+    text = _apply_pages(text, pages)
+
+    if output == "chunks":
+        return {"output": "chunks", "chunks": _to_chunks(text, max_chars), "truncated": False}
+
+    tables: List[Dict[str, Any]] = []
+    if wants_tables:
+        try:
+            tables = [
+                _compact_table(t)
+                for t in _docling_structured(path, ocr_enabled=ocr_enabled, include_tables=True)["tables"]
+            ]
+        except Exception:
+            tables = []
+    bounded, truncated = _bounded(text)
+    payload: Dict[str, Any] = {"output": output, "content": bounded, "truncated": truncated}
+    if tables:
+        payload["tables"] = tables
+    return payload
+
+
+def _bounded(text: str) -> tuple[str, bool]:
+    """Return the FULL extracted text (never truncated here); the view is bounded in ``bound_parse_payload``.
+
+    Parsing keeps the complete text so the persisted ``data`` artifact is the full parse;
+    ``max_chars`` and the default head+tail window now bound only the LLM-facing view.
+    """
+    return text, False

docsgpt/parser/embedding_pipeline.py

@@ -0,0 +1,479 @@
+import os
+import logging
+from typing import Any, List, Optional
+from retry import retry
+from tqdm import tqdm
+from docsgpt.core.settings import settings
+from docsgpt.events.publisher import publish_user_event
+from docsgpt.parser.file.base_parser import DocumentParseError
+from docsgpt.storage.db.repositories.ingest_chunk_progress import (
+    IngestChunkProgressRepository,
+)
+from docsgpt.storage.db.session import db_session
+from docsgpt.vectorstore.vector_creator import VectorCreator
+
+
+class EmbeddingPipelineError(Exception):
+    """Raised when the per-chunk embed loop produces a partial index.
+
+    Escapes into Celery's ``autoretry_for`` so a transient cause (rate
+    limit, network blip) gets another shot. The chunk-progress
+    checkpoint makes retries cheap — only the failed-and-after chunks
+    re-run. After ``MAX_TASK_ATTEMPTS`` the poison-loop guard in
+    ``with_idempotency`` finalises the row as ``failed``.
+    """
+
+
+def sanitize_content(content: str) -> str:
+    """
+    Remove NUL characters that can cause vector store ingestion to fail.
+    
+    Args:
+        content (str): Raw content that may contain NUL characters
+        
+    Returns:
+        str: Sanitized content with NUL characters removed
+    """
+    if not content:
+        return content
+    return content.replace('\x00', '')
+
+
+# Fallback when ``settings.EMBEDDINGS_BATCH_SIZE`` is unset or unusable.
+DEFAULT_EMBEDDINGS_BATCH_SIZE = 32
+
+
+def _resolve_batch_size() -> int:
+    """Read ``EMBEDDINGS_BATCH_SIZE``, falling back to the default.
+
+    Tolerates a missing, ``None`` or non-numeric setting (tests patch
+    ``settings`` with a ``MagicMock``) so a misconfiguration degrades to
+    the default batch rather than breaking ingest.
+
+    Returns:
+        Chunks per embed request, always >= 1.
+    """
+    raw = getattr(settings, "EMBEDDINGS_BATCH_SIZE", None)
+    # Explicit type check rather than a bare ``int(raw)``: ``int(MagicMock())``
+    # succeeds and yields 1, which would silently drop ingest back to the
+    # per-chunk behaviour this batching replaces.
+    if isinstance(raw, bool) or not isinstance(raw, (int, str)):
+        return DEFAULT_EMBEDDINGS_BATCH_SIZE
+    try:
+        size = int(raw)
+    except (TypeError, ValueError):
+        return DEFAULT_EMBEDDINGS_BATCH_SIZE
+    return max(1, size)
+
+
+# Per-batch inline retry. Aggressive defaults (tries=10, delay=60) blocked
+# the loop for up to 9 min per chunk and wedged the heartbeat: lower the
+# tail so a transient failure fails-fast and the chunk-progress checkpoint
+# resumes cleanly on next dispatch.
+@retry(tries=3, delay=5, backoff=2)
+def add_texts_to_store_with_retry(
+    store: Any, docs: List[Any], source_id: str
+) -> None:
+    """Add a batch of documents to the vector store with retry logic.
+
+    One call per batch replaces one call per chunk: the remote embeddings
+    API accepts a list, and the store writes the whole batch in a single
+    transaction, so this collapses N HTTP round-trips and N INSERTs into
+    one of each. Safe to retry — ``add_texts`` embeds before it inserts
+    and rolls back on failure, so a failed batch writes nothing.
+
+    Args:
+        store: The vector store object.
+        docs: The documents to be added.
+        source_id: Unique identifier for the source.
+
+    Raises:
+        Exception: If the batch fails after all retry attempts.
+    """
+    if not docs:
+        return
+    try:
+        texts: List[str] = []
+        metadatas: List[Any] = []
+        for doc in docs:
+            # Sanitize content to remove NUL characters that cause ingestion failures
+            doc.page_content = sanitize_content(doc.page_content)
+            doc.metadata["source_id"] = str(source_id)
+            texts.append(doc.page_content)
+            metadatas.append(doc.metadata)
+        store.add_texts(texts, metadatas=metadatas)
+    except Exception as e:
+        logging.error(
+            f"Failed to add {len(docs)} document(s) with retry: {e}", exc_info=True
+        )
+        raise
+
+
+def add_text_to_store_with_retry(store: Any, doc: Any, source_id: str) -> None:
+    """Add a single document to the vector store with retry logic.
+
+    Thin wrapper over :func:`add_texts_to_store_with_retry`, kept for the
+    per-chunk fallback in the embed loop and for callers outside it.
+
+    Args:
+        store: The vector store object.
+        doc: The document to be added.
+        source_id: Unique identifier for the source.
+
+    Raises:
+        Exception: If document addition fails after all retry attempts.
+    """
+    add_texts_to_store_with_retry(store, [doc], source_id)
+
+
+def _init_progress_and_resume_index(
+    source_id: str, total_chunks: int, attempt_id: Optional[str],
+) -> int:
+    """Upsert the progress row and return the next chunk index to embed.
+
+    The repository's upsert preserves ``last_index`` only when the
+    incoming ``attempt_id`` matches the stored one (a Celery autoretry
+    of the same task). On a fresh attempt — including any caller that
+    doesn't pass an ``attempt_id``, e.g. legacy code or tests — the
+    row's checkpoint is reset so the loop starts from chunk 0. This
+    is what prevents a completed checkpoint from any prior run
+    silently no-op'ing the next sync/reingest.
+
+    Best-effort: a DB outage falls back to ``0`` (fresh run from
+    chunk 0). The embed loop's own re-raise still ensures partial
+    runs don't get cached as complete.
+    """
+    try:
+        with db_session() as conn:
+            progress = IngestChunkProgressRepository(conn).init_progress(
+                source_id, total_chunks, attempt_id,
+            )
+    except Exception as e:
+        logging.warning(
+            f"Could not init ingest progress for {source_id}: {e}",
+            exc_info=True,
+        )
+        return 0
+    if not progress:
+        return 0
+    last_index = progress.get("last_index", -1)
+    if last_index is None or last_index < 0:
+        return 0
+    return int(last_index) + 1
+
+
+def _record_progress(source_id: str, last_index: int, embedded_chunks: int) -> None:
+    """Best-effort checkpoint after each chunk; logged but never raised."""
+    try:
+        with db_session() as conn:
+            IngestChunkProgressRepository(conn).record_chunk(
+                source_id, last_index=last_index, embedded_chunks=embedded_chunks
+            )
+    except Exception as e:
+        logging.warning(
+            f"Could not record ingest progress for {source_id}: {e}", exc_info=True
+        )
+
+
+def _embed_batch_individually(
+    store: Any,
+    docs: List[Any],
+    batch_start: int,
+    batch_end: int,
+    source_id: str,
+) -> tuple[Exception | None, int | None]:
+    """Re-run one failed batch a chunk at a time, checkpointing as it goes.
+
+    Called only after a batch raised. Preserves the pre-batching failure
+    contract: chunks before the offender are embedded and recorded, and the
+    returned index is the exact chunk that failed rather than the batch head.
+
+    Args:
+        store: The vector store object.
+        docs: The full chunk list.
+        batch_start: Index of the first chunk in the failed batch.
+        batch_end: Index one past the last chunk in the failed batch.
+        source_id: Unique identifier for the source.
+
+    Returns:
+        ``(None, None)`` when every chunk succeeded on its own, else the
+        exception and the index of the chunk that failed.
+    """
+    for idx in range(batch_start, batch_end):
+        try:
+            add_text_to_store_with_retry(store, docs[idx], source_id)
+            _record_progress(source_id, last_index=idx, embedded_chunks=idx + 1)
+        except Exception as e:
+            return e, idx
+    return None, None
+
+
+def assert_index_complete(source_id: str) -> None:
+    """Raise ``EmbeddingPipelineError`` if ``ingest_chunk_progress``
+    shows a partial embed for ``source_id``.
+
+    Defense-in-depth tripwire that workers run after
+    ``embed_and_store_documents`` to catch any future swallow path
+    that bypasses the function's own re-raise — the chunk-progress
+    row is the authoritative record of how many chunks landed.
+    No-op when no row exists (zero-doc validation raised before init,
+    or progress repo was unreachable).
+    """
+    try:
+        with db_session() as conn:
+            progress = IngestChunkProgressRepository(conn).get_progress(source_id)
+    except Exception as e:
+        logging.warning(
+            f"assert_index_complete: progress lookup failed for "
+            f"{source_id}: {e}",
+            exc_info=True,
+        )
+        return
+    if not progress:
+        return
+    embedded = int(progress.get("embedded_chunks") or 0)
+    total = int(progress.get("total_chunks") or 0)
+    if embedded < total:
+        raise EmbeddingPipelineError(
+            f"partial index for source {source_id}: "
+            f"{embedded}/{total} chunks embedded"
+        )
+
+
+def embed_and_store_documents(
+    docs: List[Any],
+    folder_name: str,
+    source_id: str,
+    task_status: Any,
+    *,
+    attempt_id: Optional[str] = None,
+    user_id: Optional[str] = None,
+    progress_start: int = 0,
+    progress_end: int = 100,
+) -> None:
+    """Embeds documents and stores them in a vector store.
+
+    Resumable across Celery autoretries of the *same* task: when
+    ``attempt_id`` matches the stored checkpoint's ``attempt_id``,
+    the loop resumes from ``last_index + 1``. A different
+    ``attempt_id`` (a fresh sync / reingest invocation) resets the
+    checkpoint so the index is rebuilt from chunk 0 — this is what
+    keeps a completed checkpoint from poisoning the next sync.
+
+    Args:
+        docs: List of documents to be embedded and stored.
+        folder_name: Directory to save the vector store.
+        source_id: Unique identifier for the source.
+        task_status: Task state manager for progress updates.
+        attempt_id: Stable id of the current task invocation,
+            typically ``self.request.id`` from the Celery task body.
+            ``None`` is treated as a fresh attempt every time.
+        user_id: When provided, per-percent SSE progress events are
+            published to ``user:{user_id}`` for the in-app upload toast.
+            ``None`` is the safe default — workers without a user
+            context (e.g. background syncs) skip the publish.
+        progress_start: Percent the reported progress maps to at chunk 0.
+            Lets a caller reserve the lower band for an earlier stage
+            (e.g. parsing). Defaults to ``0`` (embed owns the whole bar).
+        progress_end: Percent the reported progress maps to at the final
+            chunk. Defaults to ``100``.
+
+    Returns:
+        None
+
+    Raises:
+        OSError: If unable to create folder or save vector store.
+        EmbeddingPipelineError: If a chunk fails after retries.
+    """
+    # Ensure the folder exists
+    if not os.path.exists(folder_name):
+        os.makedirs(folder_name)
+
+    # Drop blank documents before validating. A file that parses to nothing
+    # (empty upload, whitespace-only, an image-only PDF with no OCR) used to
+    # reach here as a one-element list of "" and ingest as a healthy source,
+    # putting an embedding of the empty string into the index.
+    docs = [
+        d for d in docs
+        if str(getattr(d, "text", getattr(d, "page_content", d)) or "").strip()
+    ]
+    if not docs:
+        # ``DocumentParseError``, not ``ValueError``: an empty or image-only
+        # file yields nothing on every attempt, so this must reach the Celery
+        # tasks' ``dont_autoretry_for`` and fail once. As a bare ``ValueError``
+        # it was swept up by ``autoretry_for=(Exception,)`` and re-failed
+        # identically across the whole backoff envelope.
+        raise DocumentParseError(
+            "No text could be extracted from this file. It may be empty, "
+            "image-only, or in an unsupported format."
+        )
+
+    total_docs = len(docs)
+    # Atomic upsert that preserves checkpoint state on attempt-id match
+    # (autoretry of same task) and resets it on mismatch (fresh sync /
+    # reingest). Returns the new resume index — 0 means "start fresh".
+    resume_index = _init_progress_and_resume_index(
+        source_id, total_docs, attempt_id,
+    )
+    is_resume = resume_index > 0
+
+    # Initialize vector store
+    if settings.VECTOR_STORE == "faiss":
+        if is_resume:
+            # Load the existing FAISS index from storage so chunks
+            # already embedded by the prior attempt survive the
+            # save_local rewrite at the end of this run.
+            store = VectorCreator.create_vectorstore(
+                settings.VECTOR_STORE,
+                source_id=source_id,
+                embeddings_key=os.getenv("EMBEDDINGS_KEY"),
+            )
+            loop_start = resume_index
+        else:
+            # FAISS requires at least one doc to construct the store;
+            # seed with ``docs[0]`` and let the loop pick up at index 1.
+            store = VectorCreator.create_vectorstore(
+                settings.VECTOR_STORE,
+                docs_init=[docs[0]],
+                source_id=source_id,
+                embeddings_key=os.getenv("EMBEDDINGS_KEY"),
+            )
+            # Record the seeded chunk so single-doc ingests don't fail
+            # ``assert_index_complete`` — the loop never runs for
+            # ``total_docs == 1`` and would otherwise leave
+            # ``embedded_chunks`` at 0 / ``last_index`` at -1. The loop
+            # body's per-iteration ``_record_progress`` overshoots
+            # correctly for multi-chunk runs (counts seed + iterations),
+            # so writing this checkpoint up-front is a no-op for those.
+            _record_progress(source_id, last_index=0, embedded_chunks=1)
+            loop_start = 1
+    else:
+        store = VectorCreator.create_vectorstore(
+            settings.VECTOR_STORE,
+            source_id=source_id,
+            embeddings_key=os.getenv("EMBEDDINGS_KEY"),
+        )
+        # Only wipe the index on a fresh run — a resume must keep the
+        # chunks that earlier attempts already embedded.
+        if not is_resume:
+            store.delete_index()
+        loop_start = resume_index
+
+    if is_resume and loop_start >= total_docs:
+        # Nothing left to do; the loop runs zero iterations and
+        # downstream finalize logic still executes. This is only
+        # reachable on a same-attempt retry of a task whose previous
+        # attempt finished — typically a Celery acks_late redelivery
+        # after the task already returned. The ``assert_index_complete``
+        # tripwire still validates ``embedded == total`` afterwards.
+        loop_start = total_docs
+
+    # Process and embed documents, one batch per embed request. Progress is
+    # checkpointed per batch rather than per chunk, so a 3k-chunk ingest
+    # writes ~90 progress rows instead of 3k — the per-chunk bookkeeping
+    # (Neon UPDATE + Celery update_state + SSE) dominated wall-clock more
+    # than the embedding itself.
+    chunk_error: Exception | None = None
+    failed_idx: int | None = None
+    last_published_pct = -1
+    source_id_str = str(source_id)
+    progress_span = progress_end - progress_start
+    batch_size = _resolve_batch_size()
+    batch_starts = list(range(loop_start, total_docs, batch_size))
+    for batch_start in tqdm(
+        batch_starts,
+        desc="Embedding 🦖",
+        unit="batch",
+        total=len(batch_starts),
+        bar_format="{l_bar}{bar}| Time Left: {remaining}",
+    ):
+        batch_end = min(batch_start + batch_size, total_docs)
+        last_idx = batch_end - 1
+        try:
+            # Map the embed loop into [progress_start, progress_end].
+            progress = progress_start + int(
+                (batch_end / total_docs) * progress_span
+            )
+            task_status.update_state(state="PROGRESS", meta={"current": progress})
+
+            # SSE push for sub-second upload-toast updates. Throttled to one
+            # event per percent so a 10k-chunk ingest emits ~100 events,
+            # not 10k. The Celery update_state above stays the source of
+            # truth for the polling-fallback path.
+            if user_id and progress > last_published_pct:
+                publish_user_event(
+                    user_id,
+                    "source.ingest.progress",
+                    {
+                        "current": progress,
+                        "total": total_docs,
+                        "embedded_chunks": batch_end,
+                        "stage": "embedding",
+                    },
+                    scope={"kind": "source", "id": source_id_str},
+                )
+                last_published_pct = progress
+
+            # Add the batch to the vector store
+            add_texts_to_store_with_retry(
+                store, docs[batch_start:batch_end], source_id
+            )
+            _record_progress(
+                source_id, last_index=last_idx, embedded_chunks=batch_end
+            )
+        except Exception as batch_exc:
+            # The batch failed as a unit and wrote nothing (``add_texts``
+            # embeds before it inserts and rolls back). Re-run it one chunk
+            # at a time so a single poison chunk — an oversized input the
+            # embeddings server rejects — costs only itself: the chunks
+            # before it still land and are checkpointed, and ``failed_idx``
+            # names the real offender instead of the batch head.
+            logging.warning(
+                f"Batch embed failed for chunks {batch_start}-{last_idx} "
+                f"({batch_exc}); retrying individually to isolate the failure"
+            )
+            chunk_error, failed_idx = _embed_batch_individually(
+                store, docs, batch_start, batch_end, source_id
+            )
+            if chunk_error is None:
+                # Every chunk passed on its own — the batch-level failure was
+                # transient (or a payload-size limit). Progress is recorded.
+                continue
+            logging.error(
+                f"Error embedding document {failed_idx}: {chunk_error}",
+                exc_info=True,
+            )
+            logging.info(
+                f"Saving progress at document {failed_idx} out of {total_docs}"
+            )
+            try:
+                store.save_local(folder_name)
+                logging.info("Progress saved successfully")
+            except Exception as save_error:
+                logging.error(f"CRITICAL: Failed to save progress: {save_error}", exc_info=True)
+                # Continue without breaking to attempt final save
+            break
+
+    # Save the vector store
+    if settings.VECTOR_STORE == "faiss":
+        try:
+            store.save_local(folder_name)
+            logging.info("Vector store saved successfully.")
+        except Exception as e:
+            logging.error(f"CRITICAL: Failed to save final vector store: {e}", exc_info=True)
+            raise OSError(f"Unable to save vector store to {folder_name}: {e}") from e
+    else:
+        logging.info("Vector store saved successfully.")
+
+    # Re-raise after the partial save: the chunks that *did* embed are
+    # flushed to disk and recorded in ``ingest_chunk_progress``, so a
+    # Celery autoretry resumes via ``_read_resume_index`` and only
+    # re-runs the failed-and-after chunks. Without the raise, the
+    # task body returns success and ``with_idempotency`` finalises
+    # ``task_dedup`` as ``completed`` for a partial index — poisoning
+    # the cache for 24h.
+    if chunk_error is not None:
+        raise EmbeddingPipelineError(
+            f"embed failure at chunk {failed_idx}/{total_docs} "
+            f"for source {source_id}"
+        ) from chunk_error

docsgpt/parser/file/__init__.py

@@ -0,0 +1 @@
+

docsgpt/parser/file/anydoc_parser.py

@@ -0,0 +1,417 @@
+"""anydoc parser.
+
+Converts documents to GitHub-Flavored Markdown with `firecrawl-anydoc
+<https://github.com/firecrawl/anydoc>`_: a single Rust extension with no ML
+models and no Python dependencies. Measured against docling on a 20-document
+corpus it was 20-160x faster per file (1.2 s vs 193 s on a 150-page annual
+report), imported in 6 ms instead of 3.9 s, and peaked at 107 MB RSS instead
+of 2.3 GB, with equivalent output on clean office files, CSV and text-layer
+PDFs.
+
+anydoc never OCRs. It *detects* a scanned or image-only PDF and raises
+``UnsupportedError`` ("OCR is required"), which is the routing point: a
+parser given a ``fallback_parser`` (docling when installed, the native OCR
+parser when OCR is on without docling, otherwise the legacy parser for the
+suffix) delegates there; without one the file fails loudly as
+``DocumentParseError`` rather than being stored empty.
+"""
+import logging
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple, Union
+
+from docsgpt.core.settings import settings
+from docsgpt.parser.file.base_parser import (
+    BaseParser,
+    DocumentParseError,
+    delegate_parse,
+    module_available,
+)
+
+logger = logging.getLogger(__name__)
+
+# A fallback parse (scanned-PDF delegation or trust-check re-parse) yielding
+# fewer stripped characters than this is an empty document in the making,
+# not content.
+_MIN_SCAN_FALLBACK_CHARS = 50
+
+# Suffixes anydoc 0.2.3 converts, verified with ``anydoc.format_from_extension``.
+# ``.epub`` is deliberately absent: it stays on ``EpubParser``.
+ANYDOC_SUFFIXES: Tuple[str, ...] = (
+    # Word processing
+    ".pdf",
+    ".docx",
+    ".docm",
+    ".doc",
+    ".odt",
+    ".rtf",
+    # Presentations
+    ".pptx",
+    ".pptm",
+    ".ppsx",
+    ".ppsm",
+    ".ppt",
+    ".pps",
+    ".pot",
+    ".odp",
+    # Spreadsheets / tabular
+    ".xlsx",
+    ".xlsm",
+    ".xlsb",
+    ".xls",
+    ".ods",
+    ".csv",
+)
+
+
+# The five formats every engine has its own parser for (docling / legacy);
+# under the anydoc engine they get that parser as the fallback. Everything
+# else in ``ANYDOC_SUFFIXES`` is anydoc-only ("gained") and is mapped to a
+# fallback-less ``AnydocParser`` in every parser map.
+_CORE_SUFFIXES = frozenset({".pdf", ".docx", ".pptx", ".xlsx", ".csv"})
+
+# Spreadsheets anydoc may refuse on its fixed, non-configurable limits
+# (2M XML nodes per part, 128 MiB decompressed per part — a ~265k-row sheet
+# trips them) that the plain tabular parsers read fine in a few hundred MB.
+# For these a ResourceLimitError is a reason to delegate, not to fail.
+_TABULAR_SUFFIXES = frozenset({".csv", ".xlsx", ".xlsm", ".xlsb", ".xls", ".ods"})
+ANYDOC_GAINED_SUFFIXES: Tuple[str, ...] = tuple(
+    suffix for suffix in ANYDOC_SUFFIXES if suffix not in _CORE_SUFFIXES
+)
+
+
+def anydoc_available() -> bool:
+    """Whether the ``anydoc`` extension can be imported."""
+    return module_available("anydoc")
+
+
+def _result_chars(result: Union[str, List[str]]) -> int:
+    """Stripped character count of a parser result (str or list of rows)."""
+    if isinstance(result, list):
+        return sum(len(str(part).strip()) for part in result)
+    return len(str(result).strip())
+
+
+def _is_docling_backed(parser: Optional[BaseParser]) -> bool:
+    """True when ``parser`` is a docling parser (worth a trust-check re-parse).
+
+    Looks through ``NativeOcrPdfParser``, which under ``OCR_BACKEND=native``
+    wraps the docling PDF parser as its ``text_parser`` and hands text-layer
+    PDFs — the only kind the trust check flags — straight to it.
+    """
+    if parser is None:
+        return False
+    try:
+        from docsgpt.parser.file.docling_parser import DoclingParser
+    except ImportError:
+        return False
+    if isinstance(parser, DoclingParser):
+        return True
+    return isinstance(getattr(parser, "text_parser", None), DoclingParser)
+
+
+class AnydocParser(BaseParser):
+    """Markdown conversion via ``anydoc.to_markdown`` with a typed fallback.
+
+    Attributes:
+        fallback_parser: Parser used when anydoc cannot convert the file
+            (scanned PDF, unsupported or malformed input). ``None`` means
+            such files raise ``DocumentParseError``.
+        last_engine: Name of the engine that produced the most recent parse:
+            ``"anydoc"``, or the fallback parser's ``last_engine`` / class
+            name when it was delegated to. Mirrors ``PdfiumTextParser`` so
+            the attachment worker records what actually ran.
+    """
+
+    def __init__(
+        self,
+        fallback_parser: Optional[BaseParser] = None,
+        parser_config: Optional[Dict] = None,
+    ) -> None:
+        super().__init__(parser_config)
+        self.fallback_parser = fallback_parser
+        self.last_engine: Optional[str] = None
+        # (path, problems) from the most recent parse whose output the trust
+        # check flagged but kept — surfaced via ``get_file_metadata`` as
+        # ``parse_warnings`` so the document carries its own caveat.
+        self._last_warnings: Optional[Tuple[Path, List[str]]] = None
+        # (path, count) of scanned pages OCR'd into the most recent parse.
+        self._last_ocr_pages: Optional[Tuple[Path, int]] = None
+
+    def _init_parser(self) -> Dict:
+        # Import for real rather than trusting ``find_spec``: a wheel whose
+        # native extension fails to load (glibc/arch mismatch) has a spec but
+        # no importable module, and that must fail here — where the ingest
+        # task treats it as a setup error — not as a bare ImportError from
+        # ``parse_file`` mid-batch, which ``load_data`` does not catch.
+        try:
+            import anydoc  # noqa: F401
+        except ImportError as exc:
+            raise ImportError(
+                "firecrawl-anydoc is required for AnydocParser. "
+                "Install it with: pip install firecrawl-anydoc"
+            ) from exc
+        fallback = self.fallback_parser
+        return {"fallback_parser": type(fallback).__name__ if fallback else None}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Convert ``file`` to Markdown.
+
+        Args:
+            file: Path to the document.
+            errors: Decoding error policy; anydoc decodes internally, so this
+                is only forwarded to the fallback parser.
+
+        Returns:
+            The document as Markdown.
+
+        Raises:
+            DocumentParseError: When neither anydoc nor the fallback could
+                produce text for the file, or anydoc hit a resource limit.
+        """
+        import anydoc
+
+        path = Path(file)
+        self._last_warnings = None
+        self._last_ocr_pages = None
+        try:
+            content = anydoc.to_markdown(str(path))
+        except anydoc.ResourceLimitError as exc:
+            # anydoc refused because the file is too expensive (declared
+            # decompression size, nesting depth, node count). For office
+            # documents a heavier engine would spend exactly what was just
+            # refused, so this is terminal. Spreadsheets are the exception:
+            # anydoc's cell/node limits sit below what ordinary business
+            # exports reach, and the tabular fallback (docling's gate hands
+            # oversized sheets to pandas/openpyxl) is lighter, not heavier.
+            if path.suffix.lower() in _TABULAR_SUFFIXES and self.fallback_parser is not None:
+                return self._delegate(path, errors, f"{type(exc).__name__}: {exc}")
+            self.last_engine = None
+            raise DocumentParseError(
+                f"Failed to parse {path.name}: {type(exc).__name__}: {exc}"
+            ) from exc
+        except anydoc.ConvertError as exc:
+            # Typed, per-document failures another engine may still handle:
+            # UnsupportedError (scanned PDF / unknown format), MalformedError,
+            # EncryptedError, MissingPartError.
+            ocr_needed = isinstance(exc, anydoc.UnsupportedError) and "OCR" in str(exc)
+            return self._delegate(
+                path, errors, f"{type(exc).__name__}: {exc}", ocr_needed=ocr_needed
+            )
+        except OSError as exc:
+            raise DocumentParseError(
+                f"Failed to parse {path.name}: the file could not be read."
+            ) from exc
+        except Exception as exc:
+            logger.error(f"anydoc failed on {path.name}: {exc}", exc_info=True)
+            raise DocumentParseError(
+                f"Failed to parse {path.name} with anydoc: {exc}"
+            ) from exc
+
+        if not content or not content.strip():
+            return self._delegate(path, errors, "anydoc produced no text")
+
+        self.last_engine = "anydoc"
+        if path.suffix.lower() == ".pdf":
+            content = self._finish_pdf(path, content, errors)
+        return content
+
+    def _finish_pdf(self, path: Path, content: str, errors: str) -> Union[str, List[str]]:
+        """Post-process a successful anydoc PDF conversion.
+
+        Three PDF-only steps:
+
+        1. Trust check (``PDF_TRUST_CHECK``): the two classes where anydoc
+           drops text *silently* — Type0 fonts without ToUnicode, and
+           CJK-declaring PDFs with CJK-less output. A flagged file re-parses
+           on the docling fallback when one is wired; otherwise the anydoc
+           output is kept and the problems ride along as ``parse_warnings``
+           metadata.
+        2. Mixed documents: anydoc only refuses a PDF when *every* page lacks
+           text, so a text document with scanned pages in it converts fine
+           and silently loses those pages. When the fallback parser can OCR
+           (it is wired with OCR on), the pages with no text layer are OCR'd
+           through it and appended (``_ocr_scanned_pages``).
+        3. ``ANYDOC_TABLEIZE`` (off by default): rewrite dot-leader /
+           whitespace table runs as GFM tables. Never applied to a docling
+           re-parse — docling emits real tables already.
+        """
+        problems = self._trust_problems(path, content)
+        if problems:
+            rerouted = self._reroute_flagged(path, errors, problems)
+            if rerouted is not None:
+                return rerouted
+            self._last_warnings = (path, problems)
+            logger.warning(
+                f"anydoc output for {path.name} failed the PDF trust check "
+                f"({'; '.join(problems)}); keeping it with parse_warnings"
+            )
+        content = self._ocr_scanned_pages(path, content)
+        if settings.ANYDOC_TABLEIZE:
+            from docsgpt.parser.file.tableize import tableize
+
+            content = tableize(content)
+        return content
+
+    def _trust_problems(self, path: Path, content: str) -> List[str]:
+        """Trust-check findings for ``content``; [] when disabled, clean, or the check errors."""
+        if not settings.PDF_TRUST_CHECK:
+            return []
+        try:
+            from docsgpt.parser.file.pdf_trust import verify_pdf_file
+
+            return verify_pdf_file(path, content)
+        except Exception:
+            # The check exists to catch silent loss; it must never turn a
+            # successful parse into a failure.
+            logger.warning(
+                f"PDF trust check errored on {path.name}; trusting the output",
+                exc_info=True,
+            )
+            return []
+
+    def _reroute_flagged(
+        self, path: Path, errors: str, problems: List[str]
+    ) -> Optional[Union[str, List[str]]]:
+        """Re-parse a trust-flagged PDF with a docling-backed fallback.
+
+        Returns the fallback's output, or None when there is no docling
+        fallback, it fails, or it comes back near-empty — the caller then
+        keeps the anydoc output.
+        Only docling is worth the re-parse: it ships Adobe's predefined
+        CMaps, which is exactly what the flagged font class needs; the
+        legacy pypdf parser does not.
+        """
+        fallback = self.fallback_parser
+        if not _is_docling_backed(fallback):
+            return None
+        logger.warning(
+            f"anydoc output for {path.name} failed the PDF trust check "
+            f"({'; '.join(problems)}); re-parsing with {type(fallback).__name__}"
+        )
+        try:
+            result = delegate_parse(fallback, path, errors)
+        except DocumentParseError:
+            logger.warning(
+                f"docling re-parse of trust-flagged {path.name} failed; "
+                "keeping the anydoc output",
+                exc_info=True,
+            )
+            return None
+        if _result_chars(result) < _MIN_SCAN_FALLBACK_CHARS:
+            # A docling pipeline dropout ('' / '<!-- image -->') returns
+            # without raising; adopting it would swap anydoc's real text for
+            # an empty document.
+            logger.warning(
+                f"docling re-parse of trust-flagged {path.name} returned almost "
+                "no text; keeping the anydoc output"
+            )
+            return None
+        self.last_engine = getattr(fallback, "last_engine", None) or type(fallback).__name__
+        return result
+
+    def _ocr_scanned_pages(self, path: Path, content: str) -> str:
+        """Append OCR text for the pages of ``path`` that have no text layer.
+
+        Runs only when the fallback parser both has OCR on and exposes
+        ``ocr_pages`` (the native OCR PDF parser, or the docling PDF parser).
+        With OCR off the fallback is the plain pypdf parser and mixed
+        documents keep today's behaviour: text pages only. Pages are OCR'd
+        one at a time so a failure on one page costs that page, not the OCR
+        of every other; the anydoc text is always kept — a partial document
+        beats a rejected upload, and the loud-failure path stays reserved
+        for fully scanned files. anydoc's PDF markdown carries no page
+        separators, so the OCR'd pages are appended after the text pages
+        rather than spliced into position.
+        """
+        fallback = self.fallback_parser
+        if not (getattr(fallback, "ocr_enabled", False) and hasattr(fallback, "ocr_pages")):
+            return content
+        from docsgpt.parser.file.ocr_parser import scanned_page_indices
+
+        indices = scanned_page_indices(path)
+        if not indices:
+            return content
+        logger.info(
+            "%s has %d page(s) without a text layer among its text pages; OCR-ing them with %s",
+            path.name,
+            len(indices),
+            type(fallback).__name__,
+        )
+        try:
+            if not fallback.parser_config_set:
+                fallback.init_parser()
+        except Exception:  # noqa: BLE001 - keep the text pages rather than fail the file
+            logger.warning(
+                "OCR of the scanned pages of %s failed; indexing the text pages only",
+                path.name,
+                exc_info=True,
+            )
+            return content
+        extra: List[str] = []
+        for index in indices:
+            try:
+                text = fallback.ocr_pages(path, [index]).get(index, "")
+            except Exception:  # noqa: BLE001 - one page's failure must not drop the others
+                logger.warning(
+                    "OCR of page %d of %s failed; skipping that page",
+                    index + 1,
+                    path.name,
+                    exc_info=True,
+                )
+                continue
+            if text and text.strip():
+                extra.append(text)
+        if not extra:
+            return content
+        self._last_ocr_pages = (path, len(extra))
+        return content.rstrip() + "\n\n" + "\n\n".join(extra) + "\n"
+
+    def get_file_metadata(self, file: Path) -> Dict:
+        """Surface trust-check findings and OCR'd page counts for the file just parsed."""
+        metadata: Dict = {}
+        last = self._last_warnings
+        if last is not None and last[0] == Path(file):
+            metadata["parse_warnings"] = list(last[1])
+        ocr = self._last_ocr_pages
+        if ocr is not None and ocr[0] == Path(file):
+            metadata["ocr_pages"] = ocr[1]
+        return metadata
+
+    def _delegate(
+        self, path: Path, errors: str, reason: str, ocr_needed: bool = False
+    ) -> Union[str, List[str]]:
+        """Hand ``path`` to the fallback parser, or fail loudly without one.
+
+        ``ocr_needed`` marks anydoc's scanned-PDF refusal ("OCR is required").
+        The fallback still runs first — docling extracts text layers anydoc
+        refuses (seen on degenerate CJK text layers) even with OCR off — but
+        when it comes back near-empty the parse fails loudly, telling the
+        user how to get OCR, instead of silently storing an empty document
+        for a scan.
+        """
+        fallback = self.fallback_parser
+        if fallback is None:
+            self.last_engine = None
+            raise DocumentParseError(f"Failed to parse {path.name}: {reason}")
+        logger.warning(
+            f"anydoc could not convert {path.name} ({reason}); "
+            f"falling back to {type(fallback).__name__}"
+        )
+        result = delegate_parse(fallback, path, errors)
+        if ocr_needed and _result_chars(result) < _MIN_SCAN_FALLBACK_CHARS:
+            self.last_engine = None
+            if getattr(fallback, "ocr_enabled", False):
+                hint = " even with OCR enabled."
+            else:
+                hint = (
+                    ". Enable OCR to ingest scans: set OCR_ENABLED=true (and "
+                    "OCR_ATTACHMENTS_ENABLED for attachments). OCR runs on the "
+                    "tesseract binary or a DeepSeek-OCR endpoint (OCR_ENGINE), or "
+                    "through the optional docling extra when installed."
+                )
+            raise DocumentParseError(
+                f"{path.name} appears to be a scanned PDF (no text layer), and "
+                f"{type(fallback).__name__} extracted almost nothing{hint}"
+            )
+        self.last_engine = getattr(fallback, "last_engine", None) or type(fallback).__name__
+        return result

docsgpt/parser/file/audio_parser.py

@@ -0,0 +1,48 @@
+from pathlib import Path
+from typing import Dict, Union
+
+from docsgpt.core.settings import settings
+from docsgpt.parser.file.base_parser import BaseParser
+from docsgpt.stt.stt_creator import STTCreator
+from docsgpt.stt.upload_limits import enforce_audio_file_size_limit
+
+
+class AudioParser(BaseParser):
+    def __init__(self, parser_config=None):
+        super().__init__(parser_config=parser_config)
+        self._transcript_metadata: Dict[str, Dict] = {}
+
+    def _init_parser(self) -> Dict:
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, list[str]]:
+        _ = errors
+        try:
+            enforce_audio_file_size_limit(file.stat().st_size)
+        except OSError:
+            pass
+        stt = STTCreator.create_stt(settings.STT_PROVIDER)
+        result = stt.transcribe(
+            file,
+            language=settings.STT_LANGUAGE,
+            timestamps=settings.STT_ENABLE_TIMESTAMPS,
+            diarize=settings.STT_ENABLE_DIARIZATION,
+        )
+
+        transcript_metadata = {
+            "transcript_duration_s": result.get("duration_s"),
+            "transcript_language": result.get("language"),
+            "transcript_provider": result.get("provider"),
+        }
+        if result.get("segments"):
+            transcript_metadata["transcript_segments"] = result["segments"]
+
+        self._transcript_metadata[str(file)] = {
+            key: value
+            for key, value in transcript_metadata.items()
+            if value not in (None, [], {})
+        }
+        return result.get("text", "")
+
+    def get_file_metadata(self, file: Path) -> Dict:
+        return self._transcript_metadata.get(str(file), {})

docsgpt/parser/file/base.py

@@ -0,0 +1,19 @@
+"""Base reader class."""
+from abc import abstractmethod
+from typing import Any, List
+
+from docsgpt.parser.schema.base import Document
+from docsgpt.vectorstore.document_class import Document as VectorDocument
+
+
+class BaseReader:
+    """Utilities for loading data from a directory."""
+
+    @abstractmethod
+    def load_data(self, *args: Any, **load_kwargs: Any) -> List[Document]:
+        """Load data from the input directory."""
+
+    def load_vector_documents(self, **load_kwargs: Any) -> List[VectorDocument]:
+        """Load data in the vector-store document format."""
+        docs = self.load_data(**load_kwargs)
+        return [d.to_vector_format() for d in docs]

docsgpt/parser/file/base_parser.py

@@ -0,0 +1,133 @@
+"""Base parser and config class."""
+
+import importlib.util
+import logging
+import sys
+from abc import abstractmethod
+from pathlib import Path
+from typing import Dict, List, Optional, Union
+
+logger = logging.getLogger(__name__)
+
+
+class DocumentParseError(Exception):
+    """A file could not be converted to text.
+
+    Raised instead of returning the failure message as the document body: a
+    parser that hands back its own traceback produces an attachment/source
+    whose "content" is an error string, which then reaches the LLM as if it
+    were the document. Callers should treat this as a failed upload and tell
+    the user, rather than storing anything.
+
+    Deterministic for a given input, so it is marked non-retryable on the
+    Celery tasks that parse (alongside ``DataError`` and
+    ``AttachmentRejectedError``) — a poison file fails once instead of
+    re-failing identically on every retry.
+    """
+
+
+class BaseParser:
+    """Base class for all parsers."""
+
+    def __init__(self, parser_config: Optional[Dict] = None):
+        """Init params."""
+        self._parser_config = parser_config
+
+    def init_parser(self) -> None:
+        """Init parser and store it."""
+        parser_config = self._init_parser()
+        self._parser_config = parser_config
+
+    @property
+    def parser_config_set(self) -> bool:
+        """Check if parser config is set."""
+        return self._parser_config is not None
+
+    @property
+    def parser_config(self) -> Dict:
+        """Check if parser config is set."""
+        if self._parser_config is None:
+            raise ValueError("Parser config not set.")
+        return self._parser_config
+
+    @abstractmethod
+    def _init_parser(self) -> Dict:
+        """Initialize the parser with the config."""
+
+    @abstractmethod
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Parse file."""
+
+    def get_file_metadata(self, file: Path) -> Dict:
+        """Return parser-specific metadata for the most recently parsed file."""
+        _ = file
+        return {}
+
+
+def module_available(name: str) -> bool:
+    """Whether top-level module ``name`` can be imported, without importing it.
+
+    Cheap enough to call while building a parser map. Honors ``sys.modules``
+    first so tests that stub a module in (or block one out with ``None``)
+    are seen the same way the import system would see them; a stub with no
+    ``__spec__`` counts as present rather than raising.
+
+    Args:
+        name: Top-level module name, e.g. ``"docling"``.
+
+    Returns:
+        True when an import of ``name`` would succeed.
+    """
+    if name in sys.modules:
+        return sys.modules[name] is not None
+    try:
+        return importlib.util.find_spec(name) is not None
+    except (ImportError, ValueError):
+        return False
+
+
+def delegate_parse(
+    parser: "BaseParser", file: Path, errors: str
+) -> Union[str, List[str]]:
+    """Run a fallback parser, normalizing its failures to ``DocumentParseError``.
+
+    Used wherever one parser hands a file to another (docling to the plain
+    tabular parsers for oversized sheets, anydoc to docling/legacy parsers
+    for files it cannot convert). Wrapping matters twice over: the Celery
+    tasks that own these paths list ``DocumentParseError`` in
+    ``dont_autoretry_for``, so a deterministic content error fails once
+    instead of retrying four times; and ``SimpleDirectoryReader.load_data``
+    catches only ``DocumentParseError``, so one poison file is skipped into
+    ``failed_files`` instead of aborting an entire multi-file ingest.
+
+    Args:
+        parser: The parser to delegate to; initialized here if needed.
+        file: Path to the file being parsed.
+        errors: Decoding error policy, forwarded to the parser.
+
+    Returns:
+        The parsed text, or list of row strings.
+
+    Raises:
+        DocumentParseError: If the fallback parser fails for any reason.
+    """
+    try:
+        if not parser.parser_config_set:
+            parser.init_parser()
+        return parser.parse_file(file, errors)
+    except DocumentParseError:
+        raise
+    except ImportError:
+        # A fallback whose dependency is missing is a deployment problem, not
+        # a bad document: let it reach the ingest task's setup-error path
+        # instead of blaming every file with "could not be read".
+        raise
+    except Exception as e:
+        logger.error(
+            f"Fallback parse of {Path(file).name} with "
+            f"{type(parser).__name__} failed: {e}",
+            exc_info=True,
+        )
+        raise DocumentParseError(
+            f"Failed to parse {Path(file).name}: the file could not be read."
+        ) from e

docsgpt/parser/file/bulk.py

@@ -0,0 +1,650 @@
+"""Simple reader that reads files of different formats from a directory."""
+import logging
+from pathlib import Path
+from typing import Callable, Dict, List, Optional, Tuple, Union
+
+from docsgpt.parser.file.base import BaseReader
+from docsgpt.parser.file.base_parser import BaseParser, DocumentParseError, module_available
+from docsgpt.parser.file.docs_parser import DocxParser, PDFParser
+from docsgpt.parser.file.epub_parser import EpubParser
+from docsgpt.parser.file.html_parser import HTMLParser
+from docsgpt.parser.file.markdown_parser import MarkdownParser
+from docsgpt.parser.file.rst_parser import RstParser
+from docsgpt.parser.file.tabular_parser import PandasCSVParser, ExcelParser
+from docsgpt.parser.file.json_parser import JSONParser
+from docsgpt.parser.file.pptx_parser import PPTXParser
+from docsgpt.parser.file.image_parser import ImageParser
+from docsgpt.parser.file.audio_parser import AudioParser
+from docsgpt.parser.schema.base import Document
+from docsgpt.stt.constants import SUPPORTED_AUDIO_EXTENSIONS
+from docsgpt.utils import num_tokens_from_string
+from docsgpt.core.settings import settings
+from docsgpt.core.optional_deps import install_hint
+
+
+def _build_audio_parser_mapping() -> Dict[str, BaseParser]:
+    return {extension: AudioParser() for extension in SUPPORTED_AUDIO_EXTENSIONS}
+
+
+def _wrap_pdf_fast_path(pdf_parser: BaseParser) -> BaseParser:
+    """Put the pypdfium2 text-layer parser in front of ``pdf_parser``.
+
+    Args:
+        pdf_parser: Parser to fall back to for PDFs with no usable text layer.
+
+    Returns:
+        BaseParser: The fast-path parser, or ``pdf_parser`` unchanged when
+        pypdfium2 is unavailable.
+    """
+    try:
+        from docsgpt.parser.file.pdfium_parser import PdfiumTextParser
+    except ImportError:
+        logging.warning(
+            "pypdfium2 is not installed; PDF attachments will use %s",
+            type(pdf_parser).__name__,
+        )
+        return pdf_parser
+    return PdfiumTextParser(
+        fallback_parser=pdf_parser,
+        min_median_chars=settings.ATTACHMENT_PDF_TEXT_MIN_MEDIAN_CHARS,
+    )
+
+
+def _gained_format_entries() -> Dict[str, BaseParser]:
+    """Anydoc-only formats (legacy/macro Office, OpenDocument, RTF) for every map.
+
+    anydoc is a core dependency, so these suffixes are parseable under both
+    engines. When anydoc is somehow missing the entries are omitted and
+    ``SimpleDirectoryReader.load_data`` rejects such files with a
+    ``DocumentParseError`` naming the install, rather than reading OLE/zip
+    bytes as text.
+    """
+    from docsgpt.parser.file.anydoc_parser import (
+        ANYDOC_GAINED_SUFFIXES,
+        AnydocParser,
+        anydoc_available,
+    )
+
+    if not anydoc_available():
+        return {}
+    return {suffix: AnydocParser() for suffix in ANYDOC_GAINED_SUFFIXES}
+
+
+def _native_ocr_parsers(
+    text_parser: Optional[BaseParser] = None,
+) -> Optional[Tuple[BaseParser, Callable[[], BaseParser]]]:
+    """PDF parser and image-parser factory for the native OCR backend.
+
+    Args:
+        text_parser: Parser for PDFs whose every page has a text layer (the
+            docling PDF parser under the docling engine); None reads text
+            layers with pypdfium2.
+
+    Returns:
+        ``(pdf_parser, image_parser_factory)``, or None when pypdfium2 or
+        Pillow is missing — both are core dependencies, so that is a broken
+        install, logged as such.
+    """
+    from docsgpt.parser.file.ocr_parser import (
+        NativeOcrImageParser,
+        NativeOcrPdfParser,
+        native_ocr_available,
+    )
+
+    if not native_ocr_available():
+        logging.error(
+            "OCR is enabled but pypdfium2/Pillow are not importable, so the native "
+            "OCR backend cannot run; scanned PDFs and images will not be OCR'd"
+        )
+        return None
+    return NativeOcrPdfParser(text_parser=text_parser), NativeOcrImageParser
+
+
+def _image_entries(factory: Callable[[], BaseParser], suffixes) -> Dict[str, BaseParser]:
+    """One parser instance per image suffix."""
+    return {suffix: factory() for suffix in suffixes}
+
+
+_LEGACY_IMAGE_SUFFIXES = (".png", ".jpg", ".jpeg")
+
+
+def _legacy_file_extractor(pdf_text_fast_path: bool = False, ocr_enabled: bool = False) -> Dict[str, BaseParser]:
+    """Parser map that needs neither docling nor anydoc.
+
+    Args:
+        pdf_text_fast_path: Put the pypdfium2 text-layer parser in front of
+            the PDF parser.
+        ocr_enabled: Use the native OCR parsers (tesseract / DeepSeek-OCR via
+            pypdfium2 + Pillow) for PDFs and images instead of the plain
+            pypdf / remote-service parsers.
+
+    Returns:
+        Dict[str, BaseParser]: Parser keyed by lower-case file suffix.
+    """
+    from docsgpt.parser.file.ocr_parser import IMAGE_SUFFIXES
+
+    pdf_parser: BaseParser = PDFParser()
+    images: Dict[str, BaseParser] = _image_entries(ImageParser, _LEGACY_IMAGE_SUFFIXES)
+    if ocr_enabled:
+        native = _native_ocr_parsers()
+        if native is not None:
+            pdf_parser, image_factory = native
+            images = _image_entries(image_factory, sorted(IMAGE_SUFFIXES))
+    if pdf_text_fast_path:
+        pdf_parser = _wrap_pdf_fast_path(pdf_parser)
+    return {
+        ".pdf": pdf_parser,
+        ".docx": DocxParser(),
+        ".csv": PandasCSVParser(),
+        ".xlsx": ExcelParser(),
+        ".epub": EpubParser(),
+        ".md": MarkdownParser(),
+        ".rst": RstParser(),
+        ".html": HTMLParser(),
+        ".xhtml": HTMLParser(),
+        ".mdx": MarkdownParser(),
+        ".json": JSONParser(),
+        ".pptx": PPTXParser(),
+        **images,
+        **_build_audio_parser_mapping(),
+        **_gained_format_entries(),
+    }
+
+
+def _docling_file_extractor(
+    ocr_enabled: bool,
+    pdf_text_fast_path: bool = False,
+    *,
+    missing_log_level: int = logging.WARNING,
+) -> Dict[str, BaseParser]:
+    """docling parser map, or the legacy map when docling is not installed.
+
+    With OCR on, ``OCR_BACKEND`` decides who OCRs: docling's own hybrid
+    pipeline, or the native parsers (``ocr_parser.py``) with the docling PDF
+    parser kept — OCR off — for PDFs that have a text layer on every page.
+    Without docling installed the legacy map takes over, itself using the
+    native OCR parsers when OCR is on.
+
+    The availability check has to happen here, not at parse time:
+    ``DoclingParser`` only imports docling inside ``_init_parser``, and the
+    ``ImportError`` that raises is not a ``DocumentParseError``, so a map
+    that handed out docling parsers on a docling-less install would abort
+    the whole ingest on its first file instead of degrading.
+
+    Args:
+        ocr_enabled: Enable OCR in the docling PDF/image parsers.
+        pdf_text_fast_path: Put the pypdfium2 text-layer parser in front of
+            the PDF parser (attachments only; see ``get_default_file_extractor``).
+        missing_log_level: Level at which a missing docling install is logged;
+            informational when docling is only the *fallback* engine.
+
+    Returns:
+        Dict[str, BaseParser]: Parser keyed by lower-case file suffix.
+    """
+    try:
+        if not module_available("docling"):
+            raise ImportError("docling is not installed")
+        from docsgpt.parser.file.docling_parser import (
+            DoclingPDFParser,
+            DoclingDocxParser,
+            DoclingPPTXParser,
+            DoclingXLSXParser,
+            DoclingHTMLParser,
+            DoclingImageParser,
+            DoclingCSVParser,
+            DoclingAsciiDocParser,
+            DoclingVTTParser,
+            DoclingXMLParser,
+        )
+    except ImportError:
+        logging.log(
+            missing_log_level,
+            "docling is not installed. Using standard parsers%s. For layout-model "
+            "parsing, install the docling extra: %s",
+            " with native OCR" if ocr_enabled else "",
+            install_hint("docling"),
+        )
+        return _legacy_file_extractor(pdf_text_fast_path, ocr_enabled=ocr_enabled)
+
+    from docsgpt.parser.file.ocr_parser import IMAGE_SUFFIXES, resolve_ocr_backend
+
+    native = None
+    if ocr_enabled and resolve_ocr_backend() == "native":
+        native = _native_ocr_parsers(text_parser=DoclingPDFParser(ocr_enabled=False))
+    if native is not None:
+        pdf_parser, image_factory = native
+        images = _image_entries(image_factory, sorted(IMAGE_SUFFIXES))
+    else:
+        pdf_parser = DoclingPDFParser(ocr_enabled=ocr_enabled)
+        # Images (with OCR) - only use Docling when OCR is enabled
+        image_factory = (
+            (lambda: DoclingImageParser(ocr_enabled=True)) if ocr_enabled else ImageParser
+        )
+        images = _image_entries(image_factory, sorted(IMAGE_SUFFIXES))
+    if pdf_text_fast_path:
+        pdf_parser = _wrap_pdf_fast_path(pdf_parser)
+    return {
+        # Documents
+        ".pdf": pdf_parser,
+        ".docx": DoclingDocxParser(),
+        ".pptx": DoclingPPTXParser(),
+        ".xlsx": DoclingXLSXParser(),
+        # Web formats
+        ".html": DoclingHTMLParser(),
+        ".xhtml": DoclingHTMLParser(),
+        # Data formats
+        ".csv": DoclingCSVParser(),
+        ".json": JSONParser(),  # Keep JSON parser (specialized handling)
+        # Text/markup formats
+        ".md": MarkdownParser(),  # Keep markdown parser (specialized handling)
+        ".mdx": MarkdownParser(),
+        ".rst": RstParser(),
+        ".adoc": DoclingAsciiDocParser(),
+        ".asciidoc": DoclingAsciiDocParser(),
+        # Images: OCR'd by the selected backend when OCR is on, else ImageParser
+        **images,
+        # Media/subtitles
+        ".vtt": DoclingVTTParser(),
+        **_build_audio_parser_mapping(),
+        # Specialized XML formats
+        ".xml": DoclingXMLParser(),
+        # Formats docling doesn't support - use standard parsers
+        ".epub": EpubParser(),
+        # Formats only anydoc reads (legacy/macro Office, OpenDocument, RTF)
+        **_gained_format_entries(),
+    }
+
+
+def _anydoc_file_extractor(ocr_enabled: bool, pdf_text_fast_path: bool = False) -> Dict[str, BaseParser]:
+    """anydoc parser map, layered over the docling (or legacy) map.
+
+    anydoc takes every format it converts; each of those parsers gets the
+    base map's parser for the same suffix as ``fallback_parser``, so a
+    scanned PDF still reaches OCR — docling's when it is installed and
+    selected, the native OCR parser otherwise — and the legacy parser when
+    OCR is off. docling, when installed, also keeps serving what anydoc
+    cannot: ``.adoc``/``.vtt``/``.xml``. Images go to whichever OCR backend
+    is active. HTML goes to the markdownify-based ``HTMLMarkdownParser``.
+
+    ``pdf_text_fast_path`` is deliberately ignored here: anydoc already reads
+    the text layer in milliseconds *and* keeps headings/tables, and does its
+    own scanned-PDF detection, so putting pypdfium2 in front of it would
+    only lose structure.
+
+    Args:
+        ocr_enabled: Enable OCR in the docling fallback/image parsers.
+        pdf_text_fast_path: Honored only when anydoc is not installed and the
+            map degrades to the base engine.
+
+    Returns:
+        Dict[str, BaseParser]: Parser keyed by lower-case file suffix.
+    """
+    from docsgpt.parser.file.anydoc_parser import (
+        ANYDOC_SUFFIXES,
+        AnydocParser,
+        anydoc_available,
+    )
+    from docsgpt.parser.file.html_parser import HTMLMarkdownParser
+
+    if not anydoc_available():
+        logging.warning(
+            "firecrawl-anydoc is not installed; DOC_PARSER_ENGINE=anydoc is using "
+            "the %s parsers instead. Install with: pip install firecrawl-anydoc",
+            "docling" if module_available("docling") else "standard",
+        )
+        return _docling_file_extractor(ocr_enabled, pdf_text_fast_path)
+
+    base = _docling_file_extractor(
+        ocr_enabled, pdf_text_fast_path=False, missing_log_level=logging.INFO
+    )
+    extractor = dict(base)
+    for suffix in ANYDOC_SUFFIXES:
+        fallback = base.get(suffix)
+        if isinstance(fallback, AnydocParser):
+            # A gained-format entry from the base map is already a
+            # fallback-less AnydocParser — anydoc delegating to anydoc would
+            # just repeat the same failure.
+            continue
+        extractor[suffix] = AnydocParser(fallback_parser=fallback)
+    extractor[".html"] = HTMLMarkdownParser()
+    extractor[".xhtml"] = HTMLMarkdownParser()
+    return extractor
+
+
+def get_default_file_extractor(
+    ocr_enabled: Optional[bool] = None,
+    pdf_text_fast_path: bool = False,
+    engine: Optional[str] = None,
+) -> Dict[str, BaseParser]:
+    """Get the default file extractor.
+
+    The engine is ``settings.DOC_PARSER_ENGINE`` unless overridden:
+
+    * ``anydoc`` (default): firecrawl-anydoc for office/PDF/tabular formats,
+      markdownify for HTML, the active OCR backend for images and as the
+      fallback for scanned PDFs; docling (if installed) for the rest anydoc
+      rejects, legacy parsers otherwise.
+    * ``docling``: the docling map, with the legacy parsers when docling is
+      not installed.
+
+    Args:
+        ocr_enabled: Enable OCR for scanned PDFs and images, through the
+            backend ``OCR_BACKEND`` resolves to. Defaults to
+            ``settings.OCR_ENABLED``.
+        pdf_text_fast_path: Read PDFs via their embedded text layer
+            (pypdfium2) instead of docling, falling back to docling for scans.
+            Off by default: it trades docling's structural markdown for speed,
+            which suits attachments (read into a prompt) but not source
+            ingestion (chunked and embedded for retrieval). A no-op under the
+            anydoc engine, which is already that fast and keeps structure.
+        engine: ``"anydoc"`` or ``"docling"``; ``None`` reads the setting.
+
+    Returns:
+        Dict[str, BaseParser]: Parser keyed by lower-case file suffix.
+    """
+    if ocr_enabled is None:
+        ocr_enabled = settings.OCR_ENABLED
+    selected = (engine or getattr(settings, "DOC_PARSER_ENGINE", None) or "anydoc")
+    selected = str(selected).strip().lower()
+    if selected == "docling":
+        return _docling_file_extractor(ocr_enabled, pdf_text_fast_path)
+    if selected != "anydoc":
+        logging.warning("Unknown DOC_PARSER_ENGINE %r; using anydoc", selected)
+    return _anydoc_file_extractor(ocr_enabled, pdf_text_fast_path)
+
+
+# For backwards compatibility
+DEFAULT_FILE_EXTRACTOR: Dict[str, BaseParser] = get_default_file_extractor()
+
+
+class SimpleDirectoryReader(BaseReader):
+    """Simple directory reader.
+
+    Can read files into separate documents, or concatenates
+    files into one document text.
+
+    Args:
+        input_dir (str): Path to the directory.
+        input_files (List): List of file paths to read (Optional; overrides input_dir)
+        exclude_hidden (bool): Whether to exclude hidden files (dotfiles).
+        errors (str): how encoding and decoding errors are to be handled,
+              see https://docs.python.org/3/library/functions.html#open
+        recursive (bool): Whether to recursively search in subdirectories.
+            False by default.
+        required_exts (Optional[List[str]]): List of required extensions.
+            Default is None.
+        file_extractor (Optional[Dict[str, BaseParser]]): A mapping of file
+            extension to a BaseParser class that specifies how to convert that file
+            to text. See DEFAULT_FILE_EXTRACTOR.
+        num_files_limit (Optional[int]): Maximum number of files to read.
+            Default is None.
+        file_metadata (Optional[Callable[str, Dict]]): A function that takes
+            in a filename and returns a Dict of metadata for the Document.
+            Default is None.
+    """
+
+    def __init__(
+            self,
+            input_dir: Optional[str] = None,
+            input_files: Optional[List] = None,
+            exclude_hidden: bool = True,
+            errors: str = "ignore",
+            recursive: bool = True,
+            required_exts: Optional[List[str]] = None,
+            file_extractor: Optional[Dict[str, BaseParser]] = None,
+            num_files_limit: Optional[int] = None,
+            file_metadata: Optional[Callable[[str], Dict]] = None,
+    ) -> None:
+        """Initialize with parameters."""
+        super().__init__()
+
+        if not input_dir and not input_files:
+            raise ValueError("Must provide either `input_dir` or `input_files`.")
+
+        self.errors = errors
+
+        self.recursive = recursive
+        self.exclude_hidden = exclude_hidden
+        # Normalize extensions to lowercase for case-insensitive matching
+        self.required_exts = (
+            [ext.lower() for ext in required_exts] if required_exts else None
+        )
+        self.num_files_limit = num_files_limit
+
+        if input_files:
+            self.input_files = []
+            for path in input_files:
+                print(path)
+                input_file = Path(path)
+                self.input_files.append(input_file)
+        elif input_dir:
+            self.input_dir = Path(input_dir)
+            self.input_files = self._add_files(self.input_dir)
+
+        self.file_extractor = file_extractor or DEFAULT_FILE_EXTRACTOR
+        self.file_metadata = file_metadata
+        # (path, message) per file skipped by ``load_data`` as unparseable.
+        self.failed_files: List[Tuple[Path, str]] = []
+
+    def _add_files(self, input_dir: Path) -> List[Path]:
+        """Add files."""
+        input_files = sorted(input_dir.iterdir())
+        new_input_files = []
+        dirs_to_explore = []
+        for input_file in input_files:
+            if input_file.is_dir():
+                if self.recursive:
+                    dirs_to_explore.append(input_file)
+            elif self.exclude_hidden and input_file.name.startswith("."):
+                continue
+            elif (
+                    self.required_exts is not None
+                    and input_file.suffix.lower() not in self.required_exts
+            ):
+                continue
+            else:
+                new_input_files.append(input_file)
+
+        for dir_to_explore in dirs_to_explore:
+            sub_input_files = self._add_files(dir_to_explore)
+            new_input_files.extend(sub_input_files)
+
+        if self.num_files_limit is not None and self.num_files_limit > 0:
+            new_input_files = new_input_files[0: self.num_files_limit]
+
+        # print total number of files added
+        logging.debug(
+            f"> [SimpleDirectoryReader] Total files added: {len(new_input_files)}"
+        )
+
+        return new_input_files
+
+    def load_data(
+        self,
+        concatenate: bool = False,
+        progress_callback: Optional[Callable[[int, int], None]] = None,
+    ) -> List[Document]:
+        """Load data from the input directory.
+
+        Args:
+            concatenate (bool): whether to concatenate all files into one document.
+                If set to True, file metadata is ignored.
+                False by default.
+            progress_callback (Optional[Callable[[int, int], None]]): Called
+                after each file is parsed with ``(files_done, total_files)``.
+                Lets callers surface parse/OCR progress before embedding
+                begins. Exceptions raised by the callback are swallowed so
+                progress reporting can never fail ingestion.
+
+        Returns:
+            List[Document]: A list of documents.
+
+        Raises:
+            DocumentParseError: if no input file could be parsed. Individual
+                unreadable files are skipped and recorded in ``failed_files``
+                so one corrupt document does not cost the caller the rest of
+                the batch; a single-file read (the attachment path) still
+                raises, since skipping there would only defer the failure to
+                an empty result.
+        """
+        data: Union[str, List[str]] = ""
+        data_list: List[str] = []
+        metadata_list = []
+        self.file_token_counts = {}
+        self.failed_files = []
+
+        total_files = len(self.input_files)
+
+        def report_progress(files_done: int) -> None:
+            if progress_callback is None:
+                return
+            try:
+                progress_callback(files_done, total_files)
+            except Exception:
+                logging.warning("load_data progress callback failed", exc_info=True)
+
+        for file_index, input_file in enumerate(self.input_files):
+            suffix_lower = input_file.suffix.lower()
+            parser_metadata = {}
+            try:
+                if suffix_lower in self.file_extractor:
+                    parser = self.file_extractor[suffix_lower]
+                    if not parser.parser_config_set:
+                        parser.init_parser()
+                    data = parser.parse_file(input_file, errors=self.errors)
+                    parser_metadata = parser.get_file_metadata(input_file)
+                else:
+                    from docsgpt.parser.file.anydoc_parser import ANYDOC_GAINED_SUFFIXES
+
+                    if suffix_lower in ANYDOC_GAINED_SUFFIXES:
+                        # Binary office formats only anydoc reads; without it
+                        # the standard read would index OLE/zip bytes as text.
+                        raise DocumentParseError(
+                            f"No parser is available for {suffix_lower} files: "
+                            "firecrawl-anydoc is not installed (pip install firecrawl-anydoc)"
+                        )
+                    # do standard read
+                    with open(input_file, "r", errors=self.errors) as f:
+                        data = f.read()
+            except DocumentParseError as e:
+                logging.warning(f"Skipping unreadable file {input_file.name}: {e}")
+                self.failed_files.append((input_file, str(e)))
+                report_progress(file_index + 1)
+                continue
+
+            # Calculate token count for this file
+            if isinstance(data, List):
+                file_tokens = sum(num_tokens_from_string(str(d)) for d in data)
+            else:
+                file_tokens = num_tokens_from_string(str(data))
+            
+            full_path = str(input_file.resolve())
+            self.file_token_counts[full_path] = file_tokens
+            
+            base_metadata = {
+                'title': input_file.name,
+                'token_count': file_tokens,
+            }
+            if parser_metadata:
+                base_metadata.update(parser_metadata)
+            
+            if hasattr(self, 'input_dir'):
+                try:
+                    relative_path = str(input_file.relative_to(self.input_dir))
+                    base_metadata['source'] = relative_path
+                except ValueError:
+                    base_metadata['source'] = str(input_file)
+            else:
+                base_metadata['source'] = str(input_file)
+
+            if self.file_metadata is not None:
+                custom_metadata = self.file_metadata(input_file.name)
+                base_metadata.update(custom_metadata)
+
+            if isinstance(data, List):
+                # Extend data_list with each item in the data list
+                data_list.extend([str(d) for d in data])
+                # copy(): chunking writes token_count into this dict in
+                # place, so a shared reference gives every chunk the last
+                # chunk's count.
+                metadata_list.extend([base_metadata.copy() for _ in data])
+            else:
+                data_list.append(str(data))
+                metadata_list.append(base_metadata.copy())
+
+            report_progress(file_index + 1)
+
+        # Every file failed: there is nothing to ingest, so this is a failed
+        # read rather than an empty one. Callers (the attachment worker, the
+        # ingest tasks) treat it as terminal and tell the user.
+        if self.failed_files and not data_list:
+            if len(self.failed_files) == 1:
+                # Single-file read (the attachment path): the parser's own
+                # message reaches the user verbatim, so don't wrap it in
+                # "None of the 1 file(s)…".
+                raise DocumentParseError(self.failed_files[0][1])
+            names = ", ".join(p.name for p, _ in self.failed_files[:5])
+            raise DocumentParseError(
+                f"None of the {len(self.failed_files)} files could be parsed: "
+                f"{names}{'…' if len(self.failed_files) > 5 else ''} "
+                f"({self.failed_files[0][1]})"
+            )
+
+        # Build directory structure if input_dir is provided
+        if hasattr(self, 'input_dir'):
+            self.directory_structure = self.build_directory_structure(self.input_dir)
+            logging.info("Directory structure built successfully")
+        else:
+            self.directory_structure = {}
+
+        if concatenate:
+            return [Document("\n".join(data_list))]
+        elif self.file_metadata is not None:
+            return [Document(d, extra_info=m) for d, m in zip(data_list, metadata_list)]
+        else:
+            return [Document(d) for d in data_list]
+
+    def build_directory_structure(self, base_path):
+        """Build a dictionary representing the directory structure.
+
+        Args:
+            base_path: The base path to start building the structure from.
+
+        Returns:
+            dict: A nested dictionary representing the directory structure.
+        """
+        import mimetypes
+        
+        def build_tree(path):
+            """Helper function to recursively build the directory tree."""
+            result = {}
+            
+            for item in path.iterdir():
+                if self.exclude_hidden and item.name.startswith('.'):
+                    continue
+                    
+                if item.is_dir():
+                    subtree = build_tree(item)
+                    if subtree:
+                        result[item.name] = subtree
+                else:
+                    if self.required_exts is not None and item.suffix.lower() not in self.required_exts:
+                        continue
+                    
+                    full_path = str(item.resolve())
+                    file_size_bytes = item.stat().st_size
+                    mime_type = mimetypes.guess_type(item.name)[0] or "application/octet-stream"
+                    
+                    file_info = {
+                        "type": mime_type,
+                        "size_bytes": file_size_bytes
+                    }
+                    
+                    if hasattr(self, 'file_token_counts') and full_path in self.file_token_counts:
+                        file_info["token_count"] = self.file_token_counts[full_path]
+                        
+                    result[item.name] = file_info
+                    
+            return result
+        
+        return build_tree(Path(base_path))

docsgpt/parser/file/constants.py

@@ -0,0 +1,102 @@
+"""Shared file-extension constants for parsing and ingestion flows."""
+
+import os
+
+from docsgpt.parser.file.anydoc_parser import ANYDOC_GAINED_SUFFIXES
+from docsgpt.stt.constants import SUPPORTED_AUDIO_EXTENSIONS
+
+
+SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS = (
+    ".rst",
+    ".md",
+    ".pdf",
+    ".txt",
+    ".docx",
+    ".csv",
+    ".epub",
+    ".html",
+    ".mdx",
+    ".json",
+    ".xlsx",
+    ".pptx",
+    # Read by the HTML parsers on every engine.
+    ".xhtml",
+    # Read by the anydoc engine (legacy/macro Office, OpenDocument, RTF).
+    # Parseable regardless of DOC_PARSER_ENGINE: anydoc is a core dependency,
+    # and both parser maps route these suffixes to it.
+    *ANYDOC_GAINED_SUFFIXES,
+)
+
+SUPPORTED_SOURCE_IMAGE_EXTENSIONS = (".png", ".jpg", ".jpeg")
+
+SUPPORTED_SOURCE_EXTENSIONS = (
+    *SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS,
+    *SUPPORTED_SOURCE_IMAGE_EXTENSIONS,
+    *SUPPORTED_AUDIO_EXTENSIONS,
+)
+
+# Suffixes the attachment path has a dedicated parser for — exactly the keys
+# of ``get_default_file_extractor()``. Kept as a literal (importing ``bulk``
+# here would drag docling into the API process), with
+# ``tests/parser/file/test_constants.py`` asserting the two agree.
+#
+# This is *not* the whole attachment allow-list: a suffix with no parser is
+# read by ``SimpleDirectoryReader``'s plain-text fallthrough, which is right
+# for a .py or a .log and catastrophic for a video — the reader "extracts"
+# megabytes of binary garbage, truncates it, and stores it with
+# ``extraction.status == "ok"``. So unparsed suffixes are admitted on
+# content instead (``upload_limits.enforce_parseable_attachment``): text
+# passes, binary is refused. Zip is deliberately absent — source ingestion
+# extracts archives, the attachment path does not, and a zip fails the
+# content check like any other binary.
+#
+# Mirrored in ``frontend/src/constants/fileUpload.ts``; update both together.
+ATTACHMENT_PARSER_EXTENSIONS = frozenset(
+    {
+        *SUPPORTED_SOURCE_EXTENSIONS,
+        ".xhtml",
+        ".adoc",
+        ".asciidoc",
+        ".tiff",
+        ".tif",
+        ".bmp",
+        ".webp",
+        ".vtt",
+        ".xml",
+    }
+    # .txt has no parser of its own — it *is* the plain-text fallthrough. It
+    # must be sniffed like any other unparsed suffix, or renaming a video to
+    # notes.txt walks straight back into the bug this gate exists for.
+    - {".txt"}
+)
+
+
+def attachment_extension(filename: str | None) -> str:
+    """Return the lower-cased extension of ``filename`` including the dot, or ``""``.
+
+    Args:
+        filename: A bare filename or path; ``None`` and empty strings yield ``""``.
+
+    Returns:
+        The last suffix in lower case (``".pdf"``), or ``""`` when there is none.
+    """
+    if not filename:
+        return ""
+    return os.path.splitext(os.path.basename(str(filename)))[1].lower()
+
+
+def has_attachment_parser(filename: str | None) -> bool:
+    """Return whether an attachment's suffix has a dedicated parser.
+
+    A False result does not mean the file is refused: it means nothing but the
+    plain-text fallthrough will read it, so it has to earn its place on
+    content. The decision is by extension only, never by the mime type a
+    browser reports (mobile pickers ignore ``accept`` and lie).
+
+    Args:
+        filename: The upload's filename.
+
+    Returns:
+        True when the suffix is in ``ATTACHMENT_PARSER_EXTENSIONS``.
+    """
+    return attachment_extension(filename) in ATTACHMENT_PARSER_EXTENSIONS

docsgpt/parser/file/docs_parser.py

@@ -0,0 +1,70 @@
+"""Docs parser.
+
+Contains parsers for docx, pdf files.
+
+"""
+from pathlib import Path
+from typing import Dict
+
+from docsgpt.parser.file.base_parser import BaseParser
+from docsgpt.core.settings import settings
+import requests
+
+class PDFParser(BaseParser):
+    """PDF parser."""
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> str:
+        """Parse file."""
+        if settings.PARSE_PDF_AS_IMAGE:
+            doc2md_service = "https://llm.arc53.com/doc2md"
+            # alternatively you can use local vision capable LLM
+            with open(file, "rb") as file_loaded:
+                files = {'file': file_loaded}
+                response = requests.post(doc2md_service, files=files, timeout=100)
+                data = response.json()["markdown"]
+            return data
+
+        try:
+            from pypdf import PdfReader
+        except ImportError:
+            raise ValueError("pypdf is required to read PDF files.")
+        text_list = []
+        with open(file, "rb") as fp:
+            # Create a PDF object
+            pdf = PdfReader(fp)
+
+            # Get the number of pages in the PDF document
+            num_pages = len(pdf.pages)
+
+            # Iterate over every page
+            for page_index in range(num_pages):
+                # Extract the text from the page
+                page = pdf.pages[page_index]
+                page_text = page.extract_text()
+                text_list.append(page_text)
+        text = "\n".join(text_list)
+
+        return text
+
+
+class DocxParser(BaseParser):
+    """Docx parser."""
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> str:
+        """Parse file."""
+        try:
+            import docx2txt
+        except ImportError:
+            raise ValueError("docx2txt is required to read Microsoft Word files.")
+
+        text = docx2txt.process(file)
+
+        return text

docsgpt/parser/file/epub_parser.py

@@ -0,0 +1,28 @@
+"""Epub parser.
+
+Contains parsers for epub files.
+"""
+
+from pathlib import Path
+from typing import Dict
+
+from docsgpt.parser.file.base_parser import BaseParser
+
+
+class EpubParser(BaseParser):
+    """Epub Parser."""
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> str:
+        """Parse file."""
+        try:
+            from fast_ebook import epub
+        except ImportError:
+            raise ValueError("`fast-ebook` is required to read Epub files.")
+
+        book = epub.read_epub(file)
+        text = book.to_markdown()
+        return text

docsgpt/parser/file/html_parser.py

@@ -0,0 +1,238 @@
+"""HTML parsers.
+
+``HTMLParser`` returns the visible text of a page (the ``fast`` engine and
+the legacy no-docling map rely on that plain-text contract).
+``HTMLMarkdownParser`` returns Markdown instead and is what the anydoc
+engine maps ``.html``/``.xhtml`` to: anydoc has no HTML support, and
+markdownify — already a dependency of the web crawler — keeps links, GFM
+tables, headings and byte-exact code blocks at a few ms per page.
+"""
+import logging
+import re
+from pathlib import Path
+from typing import Dict, Optional, Tuple, Union
+
+from docsgpt.core.settings import settings
+from docsgpt.parser.file.base_parser import BaseParser
+from docsgpt.utils import truncate_to_line_boundary
+
+logger = logging.getLogger(__name__)
+
+# The crawler's conventions (``crawler_markdown.py`` / ``read_webpage.py``),
+# so file and web ingestion produce the same Markdown shape.
+MARKDOWNIFY_OPTIONS = {"heading_style": "ATX", "newline_style": "BACKSLASH"}
+
+# Elements whose text is never document content. ``title`` is reported via
+# ``get_file_metadata`` instead of leaking in as a stray first line.
+_DROP_TAGS = ("title", "script", "style", "noscript", "template")
+# Attributes whose ``data:`` URIs would otherwise land in the Markdown as
+# link/image targets: one inline image is megabytes of base64 "text" for the
+# chunker and embedder (a 5 MB data-URI ``<img>`` measured 7 MB of output).
+_URI_ATTRIBUTES = ("src", "href", "srcset", "poster", "data")
+
+
+def html_to_markdown(html: Union[str, bytes]) -> str:
+    """Convert an HTML/XHTML document to Markdown.
+
+    Args:
+        html: The markup. Pass the raw bytes when you have them: BeautifulSoup
+            then decodes by the document's own BOM / ``<meta charset>``
+            instead of the process locale, so a windows-1250 page keeps its
+            diacritics.
+
+    Returns:
+        Markdown with runs of blank lines collapsed to one.
+    """
+    from bs4 import BeautifulSoup
+
+    return soup_to_markdown(BeautifulSoup(html, "html.parser"))
+
+
+def soup_to_markdown(soup) -> str:
+    """Convert a parsed ``BeautifulSoup`` tree to Markdown.
+
+    Destructive: the non-content elements in ``_DROP_TAGS`` (including
+    ``<title>``) are removed from ``soup``; read the title first.
+
+    Args:
+        soup: The parsed document.
+
+    Returns:
+        Markdown with runs of blank lines collapsed to one.
+    """
+    from bs4 import CData, Declaration, ProcessingInstruction
+    from markdownify import MarkdownConverter
+
+    for tag in soup.find_all(_DROP_TAGS):
+        tag.decompose()
+    # ``<?xml ...?>`` (every XHTML file), CDATA and stray declarations are
+    # not text; markdownify would emit them as the document's first line.
+    for node in soup.find_all(string=lambda s: isinstance(s, (ProcessingInstruction, CData, Declaration))):
+        node.extract()
+    for tag in soup.find_all(True):
+        for attribute in _URI_ATTRIBUTES:
+            value = tag.get(attribute)
+            if isinstance(value, list):
+                value = " ".join(value)
+            if isinstance(value, str) and "data:" in value.lower():
+                del tag[attribute]
+    markdown = MarkdownConverter(**MARKDOWNIFY_OPTIONS).convert_soup(soup)
+    return re.sub(r"\n{3,}", "\n\n", markdown).strip()
+
+
+def _soup_title(soup) -> Optional[str]:
+    """The document's ``<title>`` text, or None."""
+    if soup.title and soup.title.string:
+        return str(soup.title.string)
+    return None
+
+
+def _trim_torn_utf8_tail(data: bytes) -> bytes:
+    """Drop a trailing partial multi-byte UTF-8 sequence left by a byte cut.
+
+    A cut that lands inside a multi-byte character matters more than one
+    lost glyph: BeautifulSoup decodes strictly and, when the declared UTF-8
+    fails on the torn tail, retries as windows-1252 — and if every other
+    non-ASCII byte happens to be cp1252-valid (German umlauts are), the
+    *whole* document comes back as mojibake. Only a UTF-8 tail is
+    recognised; a single-byte page loses at most the final byte.
+
+    Args:
+        data: A head window whose last bytes may be a torn sequence.
+
+    Returns:
+        ``data`` minus any trailing incomplete UTF-8 sequence.
+    """
+    end = len(data)
+    trailing = 0
+    while end > 0 and trailing < 3 and 0x80 <= data[end - 1] < 0xC0:
+        end -= 1
+        trailing += 1
+    if end == 0:
+        return data
+    lead = data[end - 1]
+    if lead < 0xC0:
+        return data
+    need = 2 if lead < 0xE0 else 3 if lead < 0xF0 else 4
+    if trailing + 1 < need:
+        return data[: end - 1]
+    return data
+
+
+def read_markup_head(file: Path, max_bytes: int) -> bytes:
+    """Read a markup file, head-truncated to ``max_bytes`` on a line boundary.
+
+    HTML has no lightweight fallback parser to bound it by other means (the
+    way oversized CSV/XLSX go to the plain tabular parsers), and the soup +
+    markdownify tree costs ~50x the input — 30 MB measured at 1.6 GB RSS,
+    against a 100 MB upload cap. Parsing only the head keeps one upload from
+    taking the ingest worker down; the cut lands on a line boundary when one
+    is reasonably close, and the lenient HTML parser copes with the torn tail.
+
+    Args:
+        file: Path to the markup file.
+        max_bytes: Size gate; ``<= 0`` reads the whole file.
+
+    Returns:
+        The file's bytes, or its first ``max_bytes`` when it is larger.
+    """
+    with open(file, "rb") as fh:
+        if max_bytes <= 0:
+            return fh.read()
+        head = fh.read(max_bytes + 1)
+    if len(head) <= max_bytes:
+        return head
+    logger.warning(
+        f"Markup {Path(file).name} exceeds MARKUP_MAX_BYTES ({max_bytes}); "
+        f"parsing the first {max_bytes} bytes to bound memory"
+    )
+    head = head[:max_bytes]
+    if head[:2] in (b"\xff\xfe", b"\xfe\xff"):
+        # UTF-16 (BOM-declared): a byte-level line cut lands between the two
+        # bytes of a code unit and the strict decode then fails outright, so
+        # keep an even byte count and let the lenient parser take the torn
+        # tail. (Only 2-byte units matter here; a torn surrogate pair is one
+        # lost character.)
+        return head[: len(head) - (len(head) % 2)]
+    return _trim_torn_utf8_tail(truncate_to_line_boundary(head))
+
+
+class HTMLParser(BaseParser):
+    """HTML parser."""
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, list[str]]:
+        """Extract the visible text of an HTML file.
+
+        Returns the text as a string, matching the other file parsers — the
+        title is recovered separately by ``get_file_metadata``.
+        """
+        from bs4 import BeautifulSoup
+
+        with open(file, "r", errors=errors) as f:
+            soup = BeautifulSoup(f, "html.parser")
+        return soup.get_text("\n")
+
+    def get_file_metadata(self, file: Path) -> Dict:
+        """Return the document title, when the markup carries one."""
+        from bs4 import BeautifulSoup
+
+        try:
+            with open(file, "r", errors="ignore") as f:
+                soup = BeautifulSoup(f, "html.parser")
+        except OSError:
+            return {}
+        if soup.title and soup.title.string:
+            return {"title": str(soup.title.string)}
+        return {}
+
+
+class HTMLMarkdownParser(HTMLParser):
+    """HTML/XHTML to Markdown, for chunking and retrieval.
+
+    Differs from ``HTMLParser`` in three ways: the body comes back as
+    Markdown; the file is read as bytes so the document's own charset
+    declaration wins over the process locale; and input is head-truncated at
+    ``MARKUP_MAX_BYTES`` (the docling HTML path has the same gate under
+    ``DOCLING_MARKUP_MAX_BYTES``).
+    """
+
+    def __init__(self, parser_config: Optional[Dict] = None) -> None:
+        super().__init__(parser_config)
+        # (path, title) of the most recent ``parse_file``, so the metadata
+        # call that follows it does not build the soup a second time.
+        self._last_title: Optional[Tuple[Path, Optional[str]]] = None
+
+    @staticmethod
+    def _read(file: Path) -> bytes:
+        return read_markup_head(Path(file), int(settings.MARKUP_MAX_BYTES))
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, list[str]]:
+        """Convert an HTML file to Markdown.
+
+        ``errors`` is accepted for the ``BaseParser`` contract but unused:
+        decoding is done by BeautifulSoup from the raw bytes.
+        """
+        from bs4 import BeautifulSoup
+
+        _ = errors
+        soup = BeautifulSoup(self._read(file), "html.parser")
+        self._last_title = (Path(file), _soup_title(soup))
+        return soup_to_markdown(soup)
+
+    def get_file_metadata(self, file: Path) -> Dict:
+        """Return the document title, decoded the same way as the body."""
+        from bs4 import BeautifulSoup
+
+        last = self._last_title
+        if last is not None and last[0] == Path(file):
+            title = last[1]
+        else:
+            try:
+                title = _soup_title(BeautifulSoup(self._read(file), "html.parser"))
+            except OSError:
+                return {}
+        return {"title": title} if title else {}

docsgpt/parser/file/image_parser.py

@@ -0,0 +1,31 @@
+"""Image parser.
+
+Contains parser for .png, .jpg, .jpeg files.
+
+"""
+from pathlib import Path
+import requests
+from typing import Dict, Union
+
+from docsgpt.parser.file.base_parser import BaseParser
+from docsgpt.core.settings import settings
+
+
+class ImageParser(BaseParser):
+    """Image parser."""
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, list[str]]:
+        if settings.PARSE_IMAGE_REMOTE:
+            doc2md_service = "https://llm.arc53.com/doc2md"
+            # alternatively you can use local vision capable LLM
+            with open(file, "rb") as file_loaded:
+                files = {'file': file_loaded}
+                response = requests.post(doc2md_service, files=files, timeout=100)   
+                data = response.json()["markdown"] 
+        else:
+            data = ""
+        return data

docsgpt/parser/file/json_parser.py

@@ -0,0 +1,57 @@
+import json
+from typing import Any, Dict, List, Union
+from pathlib import Path
+
+from docsgpt.parser.file.base_parser import BaseParser
+
+class JSONParser(BaseParser):
+    r"""JSON (.json) parser.
+
+    Parses JSON files into a list of strings or a concatenated document.
+    It handles both JSON objects (dictionaries) and arrays (lists).
+
+    Args:
+        concat_rows (bool): Whether to concatenate all rows into one document.
+            If set to False, a Document will be created for each item in the JSON.
+            True by default.
+
+        row_joiner (str): Separator to use for joining each row.
+            Only used when `concat_rows=True`.
+            Set to "\n" by default.
+
+        json_config (dict): Options for parsing JSON. Can be used to specify options like
+        custom decoding or formatting. Set to empty dict by default.
+
+    """
+
+    def __init__(
+            self,
+            *args: Any,
+            concat_rows: bool = True,
+            row_joiner: str = "\n",
+            json_config: dict = {},
+            **kwargs: Any
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._concat_rows = concat_rows
+        self._row_joiner = row_joiner
+        self._json_config = json_config
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Parse JSON file."""
+        
+        with open(file, 'r', encoding='utf-8') as f:
+                data = json.load(f, **self._json_config)
+
+        if isinstance(data, dict):
+            data = [data]
+
+        if self._concat_rows:
+            return self._row_joiner.join([str(item) for item in data])
+        else:
+            return data

docsgpt/parser/file/markdown_parser.py

@@ -0,0 +1,145 @@
+"""Markdown parser.
+
+Contains parser for md files.
+
+"""
+import re
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple, Union, cast
+
+from docsgpt.parser.file.base_parser import BaseParser
+from docsgpt.utils import num_tokens_from_string
+
+
+class MarkdownParser(BaseParser):
+    """Markdown parser.
+
+    Extract text from markdown files.
+    Returns dictionary with keys as headers and values as the text between headers.
+
+    """
+
+    def __init__(
+            self,
+            *args: Any,
+            remove_hyperlinks: bool = True,
+            remove_images: bool = True,
+            max_tokens: int = 2048,
+            # remove_tables: bool = True,
+            **kwargs: Any,
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._remove_hyperlinks = remove_hyperlinks
+        self._remove_images = remove_images
+        self._max_tokens = max_tokens
+        # self._remove_tables = remove_tables
+
+    def tups_chunk_append(self, tups: List[Tuple[Optional[str], str]], current_header: Optional[str],
+                          current_text: str):
+        """Append to tups chunk."""
+        num_tokens = num_tokens_from_string(current_text)
+        if num_tokens > self._max_tokens:
+            chunks = [current_text[i:i + self._max_tokens] for i in range(0, len(current_text), self._max_tokens)]
+            for chunk in chunks:
+                tups.append((current_header, chunk))
+        else:
+            tups.append((current_header, current_text))
+        return tups
+
+    def markdown_to_tups(self, markdown_text: str) -> List[Tuple[Optional[str], str]]:
+        """Convert a markdown file to a dictionary.
+
+        The keys are the headers and the values are the text under each header.
+
+        """
+        markdown_tups: List[Tuple[Optional[str], str]] = []
+        lines = markdown_text.split("\n")
+
+        current_header = None
+        current_text = ""
+
+        for line in lines:
+            header_match = re.match(r"^#+\s", line)
+            if header_match:
+                if current_header is not None:
+                    if current_text == "" or None:
+                        continue
+                    markdown_tups = self.tups_chunk_append(markdown_tups, current_header, current_text)
+
+                current_header = line
+                current_text = ""
+            else:
+                current_text += line + "\n"
+        markdown_tups = self.tups_chunk_append(markdown_tups, current_header, current_text)
+
+        if current_header is not None:
+            # pass linting, assert keys are defined
+            markdown_tups = [
+                (re.sub(r"#", "", cast(str, key)).strip(), re.sub(r"<.*?>", "", value))
+                for key, value in markdown_tups
+            ]
+        else:
+            markdown_tups = [
+                (key, re.sub("\n", "", value)) for key, value in markdown_tups
+            ]
+
+        return markdown_tups
+
+    def remove_images(self, content: str) -> str:
+        """Get a dictionary of a markdown file from its path."""
+        pattern = r"!{1}\[\[(.*)\]\]"
+        content = re.sub(pattern, "", content)
+        return content
+
+    # def remove_tables(self, content: str) -> List[List[str]]:
+    #     """Convert markdown tables to nested lists."""
+    #     table_rows_pattern = r"((\r?\n){2}|^)([^\r\n]*\|[^\r\n]*(\r?\n)?)+(?=(\r?\n){2}|$)"
+    #     table_cells_pattern = r"([^\|\r\n]*)\|"
+    #
+    #     table_rows = re.findall(table_rows_pattern, content, re.MULTILINE)
+    #     table_lists = []
+    #     for row in table_rows:
+    #         cells = re.findall(table_cells_pattern, row[2])
+    #         cells = [cell.strip() for cell in cells if cell.strip()]
+    #         table_lists.append(cells)
+    #     return str(table_lists)
+
+    def remove_hyperlinks(self, content: str) -> str:
+        """Get a dictionary of a markdown file from its path."""
+        pattern = r"\[(.*?)\]\((.*?)\)"
+        content = re.sub(pattern, r"\1", content)
+        return content
+
+    def _init_parser(self) -> Dict:
+        """Initialize the parser with the config."""
+        return {}
+
+    def parse_tups(
+            self, filepath: Path, errors: str = "ignore"
+    ) -> List[Tuple[Optional[str], str]]:
+        """Parse file into tuples."""
+        with open(filepath, "r") as f:
+            content = f.read()
+        if self._remove_hyperlinks:
+            content = self.remove_hyperlinks(content)
+        if self._remove_images:
+            content = self.remove_images(content)
+        # if self._remove_tables:
+        #     content = self.remove_tables(content)
+        markdown_tups = self.markdown_to_tups(content)
+        return markdown_tups
+
+    def parse_file(
+            self, filepath: Path, errors: str = "ignore"
+    ) -> Union[str, List[str]]:
+        """Parse file into string."""
+        tups = self.parse_tups(filepath, errors=errors)
+        results = []
+        # TODO: don't include headers right now
+        for header, value in tups:
+            if header is None:
+                results.append(value)
+            else:
+                results.append(f"\n\n{header}\n{value}")
+        return results

docsgpt/parser/file/ocr_parser.py

@@ -0,0 +1,874 @@
+"""Native OCR parsers: scanned PDFs and images without docling.
+
+OCR in DocsGPT has two backends, selected by ``OCR_BACKEND``:
+
+* ``docling`` — the layout-model pipeline in ``docling_parser.py``. Hybrid
+  OCR (only the bitmap regions of a page), reading-order and table
+  structure recovery, and five engines (tesseract / auto / ocrmac /
+  rapidocr / deepseek). Costs the optional docling install (torch, ONNX
+  models, gigabytes of image) and seconds to minutes per file.
+* ``native`` (this module) — page rendering with pypdfium2 and Pillow, both
+  core dependencies, feeding one of two engines directly: the system
+  ``tesseract`` binary, or a DeepSeek-OCR model behind an OpenAI-compatible
+  endpoint (Ollama / vLLM). No ML models load in the worker. Pages that
+  carry a text layer are read through pypdfium2 and never OCR'd; pages
+  without one are rendered and OCR'd. What it lacks against docling is the
+  layout model: multi-column scans rely on tesseract's own page
+  segmentation, and tesseract yields tables as plain lines (DeepSeek-OCR
+  emits Markdown tables itself).
+
+``auto`` picks docling when it is installed and native otherwise, so a
+deployment that never installs the docling extra gets working OCR from the
+tesseract binary alone, and one that does install it keeps today's
+behaviour unchanged.
+"""
+import base64
+import functools
+import io
+import logging
+import re
+import shutil
+import subprocess
+from pathlib import Path
+from typing import Dict, List, Optional, Protocol, Tuple, Union
+
+from docsgpt.parser.file.base_parser import (
+    BaseParser,
+    DocumentParseError,
+    delegate_parse,
+    module_available,
+)
+
+from docsgpt.core.optional_deps import install_hint
+
+logger = logging.getLogger(__name__)
+
+# Every engine ``OCR_ENGINE`` accepts. ``auto``, ``ocrmac`` and ``rapidocr``
+# exist only inside docling; the native backend maps them to tesseract.
+VALID_OCR_ENGINES: Tuple[str, ...] = ("tesseract", "auto", "ocrmac", "rapidocr", "deepseek")
+NATIVE_OCR_ENGINES: Tuple[str, ...] = ("tesseract", "deepseek")
+VALID_OCR_BACKENDS: Tuple[str, ...] = ("auto", "docling", "native")
+
+IMAGE_SUFFIXES = frozenset({".png", ".jpg", ".jpeg", ".tiff", ".tif", ".bmp", ".webp"})
+
+# Characters in a page's text layer at or above which the page is read from
+# that layer instead of being OCR'd. Same separation ``PdfiumTextParser``
+# relies on: scanned pages sit at 0-17 chars, text pages in the hundreds+.
+DEFAULT_TEXT_LAYER_MIN_CHARS = 32
+# Chars-per-page floor for the near-empty guard when the setting is unusable.
+_DEFAULT_MIN_CHARS_PER_PAGE = 20
+_DEFAULT_RENDER_DPI = 200
+_MIN_RENDER_DPI, _MAX_RENDER_DPI = 72, 600
+# Pixel budget for one rendered page. pypdfium2 allocates width*height*4
+# bytes up front with no cap of its own, so a 14400x14400 pt page (the PDF
+# maximum; a 191-byte file) at 200 dpi would ask for 40000x40000 px = 6 GiB.
+# 40 MP is ~10x a Letter page at 200 dpi and still OCRs an A0 poster at a
+# usable ~125 dpi; anything larger renders at the scale that fits.
+_MAX_RENDER_PIXELS = 40_000_000
+_TESSERACT_TIMEOUT_SECONDS = 300
+# tesseract reports a missing language pack on stderr and, when at least one
+# other requested pack loads, exits 0 and silently OCRs with what it has —
+# so the exit code alone cannot catch OCR_LANGS=eng+chi_sim without chi_sim.
+_TESSERACT_LANG_ERROR_RE = re.compile(r"Error opening data file|Failed loading language")
+_TESSERACT_LANG_RE = re.compile(r"^[A-Za-z0-9_/\-]+$")
+# docling's DeepSeek-OCR prompt minus its ``<|grounding|>`` prefix: grounding
+# makes the model wrap every element in ref/det tags with bounding boxes,
+# which docling parses back into a layout tree. Plain Markdown is what the
+# ingestion pipeline stores, so ask for that directly.
+DEEPSEEK_PROMPT = "Convert the document to markdown."
+_DEEPSEEK_MAX_TOKENS = 4096
+# Defensive cleanup should a served model still emit grounding markup.
+_DEEPSEEK_DET_BLOCK_RE = re.compile(r"<\|det\|>.*?<\|/det\|>", re.DOTALL)
+_DEEPSEEK_TAG_RE = re.compile(r"<\|/?ref\|>|<\|grounding\|>|<\|end▁of▁sentence\|>")
+# tesseract's CJK models emit one space between every character ("互相 保密 协议")
+# because they segment by glyph. Chinese and Japanese are written without
+# word spaces, so collapse whitespace between two CJK characters (or CJK
+# punctuation); spaces next to Latin text, digits and line breaks stay.
+_CJK_CHAR = r"[\u3000-\u303f\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uff00-\uffef]"
+_CJK_SPACE_RE = re.compile(rf"(?<={_CJK_CHAR})[ \t]+(?={_CJK_CHAR})")
+
+
+class OcrUnavailableError(DocumentParseError):
+    """The selected OCR engine cannot run on this host (missing binary or endpoint)."""
+
+
+class OcrEngine(Protocol):
+    """Anything that turns one page image into text."""
+
+    name: str
+
+    def ocr_image(self, image) -> str:  # pragma: no cover - protocol
+        """Return the text (or Markdown) recognised in ``image`` (a PIL image)."""
+
+
+# ---------------------------------------------------------------------------
+# Resolution
+# ---------------------------------------------------------------------------
+
+
+def native_ocr_available() -> bool:
+    """Whether the native backend's rendering stack (pypdfium2 + Pillow) is importable."""
+    return module_available("pypdfium2") and module_available("PIL")
+
+
+def resolve_ocr_backend(requested: Optional[str] = None) -> str:
+    """Resolve ``OCR_BACKEND`` to the backend that will actually run.
+
+    Args:
+        requested: ``auto`` | ``docling`` | ``native``, or None to read the setting.
+
+    Returns:
+        ``"docling"`` or ``"native"``. ``auto`` prefers docling when it is
+        installed; ``docling`` without the install degrades to native with a
+        warning rather than leaving OCR off.
+    """
+    from docsgpt.core.settings import settings
+
+    backend = str(requested or getattr(settings, "OCR_BACKEND", None) or "auto").strip().lower()
+    if backend not in VALID_OCR_BACKENDS:
+        logger.warning(f"Unknown OCR_BACKEND {backend!r}; using auto")
+        backend = "auto"
+    docling_installed = module_available("docling")
+    if backend == "docling" and not docling_installed:
+        logger.warning(
+            "OCR_BACKEND=docling but docling is not installed (%s); using the native OCR backend",
+            install_hint("docling"),
+        )
+        return "native"
+    if backend == "auto":
+        return "docling" if docling_installed else "native"
+    return backend
+
+
+def resolve_native_ocr_engine(requested: Optional[str] = None) -> str:
+    """Map ``OCR_ENGINE`` onto an engine the native backend implements.
+
+    Args:
+        requested: Engine name, or None to read ``settings.OCR_ENGINE``.
+
+    Returns:
+        ``"tesseract"`` or ``"deepseek"``. docling-only engines (``auto``,
+        ``ocrmac``, ``rapidocr``) and unknown names become tesseract with a
+        warning, so switching a deployment off docling never silently
+        disables OCR.
+    """
+    from docsgpt.core.settings import settings
+
+    engine = str(requested or getattr(settings, "OCR_ENGINE", None) or "tesseract").strip().lower()
+    if engine in NATIVE_OCR_ENGINES:
+        return engine
+    if engine in VALID_OCR_ENGINES:
+        logger.warning(
+            f"OCR_ENGINE={engine!r} is only available through the docling backend; "
+            "the native backend is using tesseract"
+        )
+    else:
+        logger.warning(f"Unknown OCR_ENGINE {engine!r}; the native backend is using tesseract")
+    return "tesseract"
+
+
+def ocr_min_chars_per_page() -> int:
+    """Chars-per-page floor for the near-empty OCR guard (``OCR_MIN_CHARS_PER_PAGE``); 0 disables it."""
+    from docsgpt.core.settings import settings
+
+    try:
+        return int(getattr(settings, "OCR_MIN_CHARS_PER_PAGE", _DEFAULT_MIN_CHARS_PER_PAGE))
+    except (TypeError, ValueError):
+        return _DEFAULT_MIN_CHARS_PER_PAGE
+
+
+def render_dpi() -> int:
+    """Rendering resolution for pages that need OCR (``OCR_RENDER_DPI``), clamped to a sane range."""
+    from docsgpt.core.settings import settings
+
+    try:
+        dpi = int(getattr(settings, "OCR_RENDER_DPI", _DEFAULT_RENDER_DPI))
+    except (TypeError, ValueError):
+        dpi = _DEFAULT_RENDER_DPI
+    return max(_MIN_RENDER_DPI, min(_MAX_RENDER_DPI, dpi))
+
+
+def _has_alpha(image) -> bool:
+    return "A" in image.getbands() or (image.mode == "P" and "transparency" in image.info)
+
+
+def _png_bytes(image) -> bytes:
+    """Encode a PIL image as PNG, flattening modes the engines cannot take.
+
+    Transparency is composited onto white: a plain ``convert("RGB")`` drops
+    the alpha channel and leaves transparent pixels black, which turns dark
+    text on a transparent background into an all-black page that OCRs to
+    nothing.
+    """
+    from PIL import Image
+
+    if _has_alpha(image):
+        rgba = image.convert("RGBA")
+        canvas = Image.new("RGB", rgba.size, "white")
+        canvas.paste(rgba, mask=rgba.getchannel("A"))
+        image = canvas
+    elif image.mode not in ("RGB", "L"):
+        image = image.convert("RGB")
+    buffer = io.BytesIO()
+    image.save(buffer, format="PNG")
+    return buffer.getvalue()
+
+
+def fit_to_pixel_budget(image):
+    """Downscale a decoded image to ``_MAX_RENDER_PIXELS`` before OCR.
+
+    The same budget the PDF renderer applies: a 28 KB PNG can decode to
+    81 MP (Pillow's bomb guard only trips at ~178 MP), and every extra
+    pixel is paid again in the PNG re-encode and inside tesseract.
+    """
+    from PIL import Image
+
+    width, height = image.size
+    scale = render_scale(width, height, 72)
+    if scale >= 1.0:
+        return image
+    if image.mode not in ("RGB", "L", "RGBA", "LA"):
+        image = image.convert("RGBA" if _has_alpha(image) else "RGB")
+    logger.warning(
+        "Image is %dx%d px; downscaling to %.0f%% to stay within %d MP before OCR",
+        width,
+        height,
+        scale * 100,
+        _MAX_RENDER_PIXELS // 1_000_000,
+    )
+    return image.resize(
+        (max(1, int(width * scale)), max(1, int(height * scale))), Image.Resampling.BILINEAR
+    )
+
+
+@functools.lru_cache(maxsize=1)
+def tesseract_languages() -> Optional[frozenset]:
+    """Language packs the ``tesseract`` binary reports (``--list-langs``), or None when unknown.
+
+    Cached for the process: packs are installed with the image, not at runtime.
+    """
+    if shutil.which("tesseract") is None:
+        return None
+    try:
+        completed = subprocess.run(
+            ["tesseract", "--list-langs"], capture_output=True, timeout=30, check=False
+        )
+    except (OSError, subprocess.TimeoutExpired):
+        return None
+    output = completed.stdout.decode("utf-8", "replace") + "\n" + completed.stderr.decode("utf-8", "replace")
+    langs = frozenset(
+        line.strip() for line in output.splitlines() if _TESSERACT_LANG_RE.match(line.strip())
+    )
+    return langs or None
+
+
+# ---------------------------------------------------------------------------
+# Engines
+# ---------------------------------------------------------------------------
+
+
+class TesseractEngine:
+    """OCR through the system ``tesseract`` binary (stdin -> stdout, no Python wrapper).
+
+    Attributes:
+        languages: Tesseract language codes; None reads ``OCR_LANGS`` per call.
+        psm: Page segmentation mode (3 = fully automatic, tesseract's default).
+        timeout: Seconds allowed per page.
+    """
+
+    name = "tesseract"
+
+    def __init__(
+        self,
+        languages: Optional[List[str]] = None,
+        psm: int = 3,
+        timeout: float = _TESSERACT_TIMEOUT_SECONDS,
+    ) -> None:
+        self.languages = languages
+        self.psm = psm
+        self.timeout = timeout
+
+    @staticmethod
+    def available() -> bool:
+        """Whether a ``tesseract`` binary is on PATH."""
+        return shutil.which("tesseract") is not None
+
+    def _language_arg(self) -> str:
+        if self.languages:
+            langs = [lang.strip() for lang in self.languages if lang and lang.strip()]
+        else:
+            from docsgpt.core.settings import settings
+
+            configured = str(getattr(settings, "OCR_LANGS", "") or "eng")
+            langs = [lang.strip() for lang in configured.split("+") if lang.strip()]
+        return "+".join(langs) or "eng"
+
+    def command(self) -> List[str]:
+        """The tesseract command line, for logging and tests."""
+        return ["tesseract", "stdin", "stdout", "-l", self._language_arg(), "--psm", str(self.psm)]
+
+    def ocr_image(self, image) -> str:
+        """OCR one PIL image.
+
+        Raises:
+            OcrUnavailableError: No tesseract binary on PATH.
+            DocumentParseError: tesseract failed or timed out.
+        """
+        if not self.available():
+            raise OcrUnavailableError(
+                "OCR_ENGINE=tesseract but no tesseract binary is on PATH. Install "
+                "tesseract-ocr plus language packs (Docker: build with "
+                "INSTALL_TESSERACT=true; local: apt/brew install tesseract-ocr), "
+                "or set OCR_ENGINE=deepseek / install the docling extra."
+            )
+        try:
+            completed = subprocess.run(
+                self.command(),
+                input=_png_bytes(image),
+                capture_output=True,
+                timeout=self.timeout,
+                check=False,
+            )
+        except subprocess.TimeoutExpired as exc:
+            raise DocumentParseError(f"tesseract timed out after {self.timeout:.0f}s on a page") from exc
+        except OSError as exc:
+            raise DocumentParseError(f"tesseract could not be started: {exc}") from exc
+        stderr = completed.stderr.decode("utf-8", "replace").strip()
+        if _TESSERACT_LANG_ERROR_RE.search(stderr):
+            # Exit code 0 here means tesseract dropped the missing pack and
+            # OCR'd with the rest: Chinese scans would come out as garbage
+            # with no signal. Fail the file and name the fix instead.
+            raise OcrUnavailableError(
+                f"tesseract could not load a language pack for OCR_LANGS={self._language_arg()!r}: "
+                f"{stderr[:300]}. Install the tesseract-ocr-<lang> package for every language listed."
+            )
+        if completed.returncode != 0:
+            raise DocumentParseError(f"tesseract failed (exit {completed.returncode}): {stderr[:300]}")
+        return collapse_cjk_spaces(completed.stdout.decode("utf-8", "replace")).strip()
+
+
+def collapse_cjk_spaces(text: str) -> str:
+    """Remove the per-glyph spaces tesseract inserts inside CJK runs."""
+    return _CJK_SPACE_RE.sub("", text)
+
+
+def clean_deepseek_output(text: str) -> str:
+    """Strip DeepSeek-OCR grounding markup, keeping the referenced text."""
+    text = _DEEPSEEK_DET_BLOCK_RE.sub("", text)
+    return _DEEPSEEK_TAG_RE.sub("", text).strip()
+
+
+class DeepseekOcrEngine:
+    """DeepSeek-OCR over an OpenAI-compatible chat-completions endpoint.
+
+    One request per page, sequentially. The image travels as a base64 data
+    URL, which both Ollama and vLLM accept. Pages are sent one at a time
+    because the model server, not the worker, is the bottleneck; docling's
+    default of four concurrent requests and a 90 s timeout is what broke on
+    a laptop-hosted Ollama in testing.
+
+    Attributes:
+        url: Chat-completions URL (``OCR_DEEPSEEK_URL``).
+        model: Model name at that endpoint (``OCR_DEEPSEEK_MODEL``).
+        timeout: Seconds per page (``OCR_DEEPSEEK_TIMEOUT``).
+        prompt: Instruction sent with every page image.
+    """
+
+    name = "deepseek"
+
+    def __init__(
+        self,
+        url: Optional[str] = None,
+        model: Optional[str] = None,
+        timeout: Optional[float] = None,
+        prompt: str = DEEPSEEK_PROMPT,
+        max_tokens: int = _DEEPSEEK_MAX_TOKENS,
+    ) -> None:
+        from docsgpt.core.settings import settings
+
+        self.url = url or settings.OCR_DEEPSEEK_URL
+        self.model = model or settings.OCR_DEEPSEEK_MODEL
+        self.timeout = float(timeout if timeout is not None else getattr(settings, "OCR_DEEPSEEK_TIMEOUT", 300))
+        self.prompt = prompt
+        self.max_tokens = max_tokens
+
+    def payload(self, image) -> Dict:
+        """The chat-completions request body for one page image."""
+        data_url = "data:image/png;base64," + base64.b64encode(_png_bytes(image)).decode("ascii")
+        return {
+            "model": self.model,
+            "messages": [
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "image_url", "image_url": {"url": data_url}},
+                        {"type": "text", "text": self.prompt},
+                    ],
+                }
+            ],
+            "max_tokens": self.max_tokens,
+            "temperature": 0,
+        }
+
+    def ocr_image(self, image) -> str:
+        """OCR one PIL image through the endpoint.
+
+        Raises:
+            DocumentParseError: The request failed, timed out, or the
+                response was not a chat completion.
+        """
+        import requests
+
+        try:
+            response = requests.post(self.url, json=self.payload(image), timeout=self.timeout)
+            response.raise_for_status()
+            body = response.json()
+        except requests.exceptions.JSONDecodeError as exc:
+            # Subclasses RequestException too, so it must be caught first or a
+            # proxy's HTML error page is reported as a connection failure.
+            raise DocumentParseError(f"DeepSeek-OCR endpoint {self.url} returned a non-JSON body") from exc
+        except requests.RequestException as exc:
+            raise DocumentParseError(
+                f"DeepSeek-OCR request to {self.url} failed: {exc}. Check OCR_DEEPSEEK_URL "
+                f"and that model {self.model!r} is served (e.g. `ollama pull {self.model}`)."
+            ) from exc
+        try:
+            content = body["choices"][0]["message"]["content"]
+        except (KeyError, IndexError, TypeError) as exc:
+            raise DocumentParseError(
+                f"DeepSeek-OCR endpoint {self.url} returned no chat completion: {str(body)[:200]}"
+            ) from exc
+        if isinstance(content, list):  # some servers return content parts
+            content = "".join(part.get("text", "") for part in content if isinstance(part, dict))
+        return clean_deepseek_output(str(content or ""))
+
+
+def build_native_ocr_engine(engine: Optional[str] = None, languages: Optional[List[str]] = None) -> OcrEngine:
+    """Instantiate the engine ``OCR_ENGINE`` (or ``engine``) resolves to for the native backend."""
+    resolved = resolve_native_ocr_engine(engine)
+    if resolved == "deepseek":
+        return DeepseekOcrEngine()
+    return TesseractEngine(languages=languages)
+
+
+# ---------------------------------------------------------------------------
+# Parsers
+# ---------------------------------------------------------------------------
+
+
+def render_scale(page_width_pt: float, page_height_pt: float, dpi: int) -> float:
+    """Render scale for a page of the given size at ``dpi``, capped by ``_MAX_RENDER_PIXELS``.
+
+    The cap is what stands between an oversized (or hostile) MediaBox and a
+    multi-gigabyte bitmap allocation in the ingest worker.
+    """
+    scale = dpi / 72.0
+    area = max(1.0, float(page_width_pt)) * max(1.0, float(page_height_pt))
+    max_scale = (_MAX_RENDER_PIXELS / area) ** 0.5
+    return min(scale, max_scale)
+
+
+def _render_page(page, dpi: int):
+    """Render a pypdfium2 page to a PIL image at ``dpi``, within the pixel budget."""
+    width_pt, height_pt = page.get_size()
+    scale = render_scale(width_pt, height_pt, dpi)
+    if scale < dpi / 72.0:
+        logger.warning(
+            "Page is %.0fx%.0f pt; rendering at %.0f dpi instead of %d to stay within %d MP",
+            width_pt,
+            height_pt,
+            scale * 72.0,
+            dpi,
+            _MAX_RENDER_PIXELS // 1_000_000,
+        )
+    return page.render(scale=scale).to_pil()
+
+
+def text_layer_counts(path: Path) -> List[int]:
+    """Characters in each page's text layer (pypdfium2), in page order.
+
+    Raises:
+        Exception: Propagated from pypdfium2 when the file cannot be opened.
+    """
+    import pypdfium2 as pdfium
+
+    pdf = pdfium.PdfDocument(str(path))
+    counts: List[int] = []
+    try:
+        for index in range(len(pdf)):
+            page = pdf[index]
+            try:
+                textpage = page.get_textpage()
+                try:
+                    counts.append(max(0, textpage.count_chars()))
+                finally:
+                    textpage.close()
+            finally:
+                page.close()
+    finally:
+        pdf.close()
+    return counts
+
+
+def _page_has_image(page) -> bool:
+    """Whether the page draws at least one image XObject (something OCR could read)."""
+    import pypdfium2.raw as pdfium_c
+
+    try:
+        # pypdfium2's default max_depth=2 misses images inside nested Form
+        # XObjects, which print drivers and InDesign produce routinely.
+        objects = page.get_objects(filter=(pdfium_c.FPDF_PAGEOBJ_IMAGE,), max_depth=16)
+        return next(iter(objects), None) is not None
+    except Exception:  # noqa: BLE001 - treat an unreadable page as image-bearing
+        return True
+
+
+def scanned_page_indices(path: Path, min_chars: int = DEFAULT_TEXT_LAYER_MIN_CHARS) -> List[int]:
+    """0-based indices of the pages that look scanned: no text layer, or a thin one over an image.
+
+    The probe behind mixed-document handling: a text-layer converter (anydoc)
+    reads the text pages and silently skips these, so whoever owns OCR has to
+    fill them in. A page with no text at all always qualifies. A page with a
+    thin text layer (under ``min_chars``: a Bates stamp on a scan, a page
+    number under a full-page figure) qualifies only when it also draws an
+    image — a cover page reading just "Annual Report 2025" has nothing to OCR,
+    and OCR-ing it would duplicate text the converter already read. Returns []
+    when the file cannot be probed — the caller then keeps what it has rather
+    than failing a successful parse.
+    """
+    import pypdfium2 as pdfium
+
+    floor = max(1, int(min_chars))
+    indices: List[int] = []
+    try:
+        pdf = pdfium.PdfDocument(str(path))
+        try:
+            for index in range(len(pdf)):
+                page = pdf[index]
+                try:
+                    textpage = page.get_textpage()
+                    try:
+                        count = max(0, textpage.count_chars())
+                    finally:
+                        textpage.close()
+                    if count == 0 or (count < floor and _page_has_image(page)):
+                        indices.append(index)
+                finally:
+                    page.close()
+        finally:
+            pdf.close()
+    except Exception:  # noqa: BLE001 - a probe must never fail the parse
+        logger.warning("Could not probe %s for scanned pages", Path(path).name, exc_info=True)
+        return []
+    return indices
+
+
+def _check_near_empty(name: str, engine: str, content: str, pages: int, ocr_pages: int) -> None:
+    """Loud failure for OCR that produced nothing, a warning for text-sparse documents.
+
+    Mirrors the docling parser's dropout guard: failing only on positive
+    evidence — several pages that OCR'd to literally nothing — because a
+    photo, logo or chart is genuinely text-poor, and ``DocumentParseError``
+    rejects the upload permanently.
+    """
+    if ocr_pages == 0:
+        return
+    floor = ocr_min_chars_per_page()
+    chars = len(content.strip())
+    if floor <= 0 or chars >= floor * pages:
+        return
+    if chars == 0 and pages > 1:
+        raise DocumentParseError(
+            f"OCR ({engine}) produced no text over {pages} pages of {name}; "
+            "document not indexed. Check the engine's languages (OCR_LANGS) and "
+            "that the scan is legible."
+        )
+    logger.warning(
+        "%s OCR'd (%s) to only %d chars over %d page(s); indexing it as text-sparse",
+        name,
+        engine,
+        chars,
+        pages,
+    )
+
+
+class NativeOcrPdfParser(BaseParser):
+    """PDF parser that OCRs pages without a text layer and reads the rest directly.
+
+    Per page: a text layer of at least ``min_text_chars`` characters is read
+    with pypdfium2; anything thinner is rendered and sent to the engine. So
+    a scan, a born-digital PDF and a mix of both all come out right without
+    any prior classification.
+
+    Attributes:
+        engine: The ``OcrEngine``; built from ``OCR_ENGINE`` on first use when None.
+        text_parser: Optional parser for documents whose *every* page has a
+            text layer — used under ``DOC_PARSER_ENGINE=docling`` to keep
+            docling's structured Markdown for text PDFs while scans take the
+            native OCR path. None reads text layers with pypdfium2.
+        min_text_chars: Text-layer characters per page below which the page is OCR'd.
+        ocr_enabled: Always True; lets callers that inspect their fallback
+            parser (``AnydocParser``) phrase their errors correctly.
+        last_engine: Engine name behind the most recent parse.
+    """
+
+    ocr_enabled = True
+
+    def __init__(
+        self,
+        engine: Optional[OcrEngine] = None,
+        text_parser: Optional[BaseParser] = None,
+        min_text_chars: int = DEFAULT_TEXT_LAYER_MIN_CHARS,
+        parser_config: Optional[Dict] = None,
+    ) -> None:
+        super().__init__(parser_config)
+        self.engine = engine
+        self.text_parser = text_parser
+        self.min_text_chars = max(1, int(min_text_chars))
+        self.last_engine: Optional[str] = None
+        self._last_metadata: Dict = {}
+
+    def _init_parser(self) -> Dict:
+        try:
+            import pypdfium2  # noqa: F401
+            from PIL import Image  # noqa: F401
+        except ImportError as exc:
+            raise ImportError("pypdfium2 and Pillow are required for NativeOcrPdfParser") from exc
+        if self.engine is None:
+            self.engine = build_native_ocr_engine()
+        return {
+            "engine": self.engine.name,
+            "text_parser": type(self.text_parser).__name__ if self.text_parser else None,
+            "min_text_chars": self.min_text_chars,
+        }
+
+    def _ensure_engine(self) -> OcrEngine:
+        if self.engine is None:
+            self.engine = build_native_ocr_engine()
+        return self.engine
+
+    def _text_layer_counts(self, pdf) -> List[int]:
+        counts: List[int] = []
+        for index in range(len(pdf)):
+            page = pdf[index]
+            try:
+                textpage = page.get_textpage()
+                try:
+                    counts.append(max(0, textpage.count_chars()))
+                finally:
+                    textpage.close()
+            finally:
+                page.close()
+        return counts
+
+    def ocr_pages(self, file: Path, indices: List[int]) -> Dict[int, str]:
+        """OCR only the given 0-based pages of ``file``, ignoring their text layers.
+
+        Used by ``AnydocParser`` for mixed documents: anydoc has already read
+        the text pages, so only the scanned ones come here.
+
+        Returns:
+            Page index -> recognised text, in the order requested.
+
+        Raises:
+            DocumentParseError: The file cannot be opened or the engine failed.
+        """
+        import pypdfium2 as pdfium
+
+        path = Path(file)
+        engine = self._ensure_engine()
+        try:
+            pdf = pdfium.PdfDocument(str(path))
+        except Exception as exc:
+            raise DocumentParseError(f"Failed to open {path.name} with pypdfium2: {exc}") from exc
+        texts: Dict[int, str] = {}
+        try:
+            dpi = render_dpi()
+            for index in indices:
+                if index < 0 or index >= len(pdf):
+                    continue
+                page = pdf[index]
+                try:
+                    texts[index] = (engine.ocr_image(_render_page(page, dpi)) or "").strip()
+                finally:
+                    page.close()
+        except DocumentParseError:
+            raise
+        except Exception as exc:
+            raise DocumentParseError(f"Failed to OCR pages of {path.name} ({engine.name}): {exc}") from exc
+        finally:
+            pdf.close()
+        self.last_engine = engine.name
+        return texts
+
+    def text_layer_delegate(self, file: Path) -> Optional[BaseParser]:
+        """The ``text_parser`` that ``parse_file`` would hand ``file`` to, or None.
+
+        None when there is no text parser, the file cannot be opened, or any
+        page falls below ``min_text_chars`` (those are OCR'd here). Callers
+        that want a by-product of the delegate's conversion (docling tables)
+        can run it directly instead of converting the document twice.
+        """
+        if self.text_parser is None:
+            return None
+        try:
+            import pypdfium2 as pdfium
+
+            pdf = pdfium.PdfDocument(str(Path(file)))
+        except Exception:  # noqa: BLE001 - a probe; parse_file reports the real error
+            return None
+        try:
+            counts = self._text_layer_counts(pdf)
+        except Exception:  # noqa: BLE001
+            return None
+        finally:
+            pdf.close()
+        if counts and all(count >= self.min_text_chars for count in counts):
+            return self.text_parser
+        return None
+
+    def _delegate_text(self, path: Path, errors: str) -> Union[str, List[str]]:
+        text_parser = self.text_parser
+        assert text_parser is not None
+        logger.info(
+            "%s has a text layer on every page; parsing with %s instead of OCR",
+            path.name,
+            type(text_parser).__name__,
+        )
+        result = delegate_parse(text_parser, path, errors)
+        self.last_engine = getattr(text_parser, "last_engine", None) or type(text_parser).__name__
+        self._last_metadata = {**text_parser.get_file_metadata(path), "parse_engine": self.last_engine}
+        return result
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Parse a PDF, OCR-ing only the pages that need it.
+
+        Raises:
+            DocumentParseError: The file cannot be opened, the engine failed,
+                or several pages OCR'd to nothing.
+        """
+        import pypdfium2 as pdfium
+
+        path = Path(file)
+        engine = self._ensure_engine()
+        try:
+            pdf = pdfium.PdfDocument(str(path))
+        except Exception as exc:
+            raise DocumentParseError(f"Failed to open {path.name} with pypdfium2: {exc}") from exc
+
+        pages_text: List[str] = []
+        ocr_pages = 0
+        try:
+            page_count = len(pdf)
+            if page_count == 0:
+                raise DocumentParseError(f"{path.name} has no pages")
+            counts = self._text_layer_counts(pdf)
+            if self.text_parser is not None and all(count >= self.min_text_chars for count in counts):
+                pdf.close()
+                pdf = None
+                return self._delegate_text(path, errors)
+            dpi = render_dpi()
+            for index in range(page_count):
+                page = pdf[index]
+                try:
+                    if counts[index] >= self.min_text_chars:
+                        textpage = page.get_textpage()
+                        try:
+                            text = textpage.get_text_bounded()
+                        finally:
+                            textpage.close()
+                    else:
+                        text = engine.ocr_image(_render_page(page, dpi))
+                        ocr_pages += 1
+                finally:
+                    page.close()
+                pages_text.append((text or "").strip())
+        except DocumentParseError:
+            raise
+        except Exception as exc:
+            raise DocumentParseError(f"Failed to parse {path.name} with native OCR ({engine.name}): {exc}") from exc
+        finally:
+            if pdf is not None:
+                pdf.close()
+
+        content = "\n\n".join(pages_text)
+        _check_near_empty(path.name, engine.name, content, page_count, ocr_pages)
+        self.last_engine = engine.name
+        self._last_metadata = {"parse_engine": engine.name, "pdf_pages": page_count, "ocr_pages": ocr_pages}
+        logger.info(
+            "Parsed %s with native OCR (%s): %d/%d page(s) OCR'd, %d chars",
+            path.name,
+            engine.name,
+            ocr_pages,
+            page_count,
+            len(content),
+        )
+        return content
+
+    def get_file_metadata(self, file: Path) -> Dict:
+        """Engine and page counts for the most recently parsed file."""
+        _ = file
+        return dict(self._last_metadata)
+
+
+class NativeOcrImageParser(BaseParser):
+    """Image parser that OCRs every frame (multi-page TIFFs included) with the native engine."""
+
+    ocr_enabled = True
+
+    def __init__(self, engine: Optional[OcrEngine] = None, parser_config: Optional[Dict] = None) -> None:
+        super().__init__(parser_config)
+        self.engine = engine
+        self.last_engine: Optional[str] = None
+        self._last_metadata: Dict = {}
+
+    def _init_parser(self) -> Dict:
+        try:
+            from PIL import Image  # noqa: F401
+        except ImportError as exc:
+            raise ImportError("Pillow is required for NativeOcrImageParser") from exc
+        if self.engine is None:
+            self.engine = build_native_ocr_engine()
+        return {"engine": self.engine.name}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """OCR an image file.
+
+        Only TIFF frames are pages (multi-page faxes and scans); every other
+        multi-frame format is an animation (GIF, WebP, APNG) whose frames
+        repeat one picture, so only the first is OCR'd — a 50-frame WebP
+        would otherwise cost 50 tesseract runs and then be rejected as an
+        empty multi-page document.
+
+        Raises:
+            DocumentParseError: The image cannot be decoded, the engine failed,
+                or a multi-frame image OCR'd to nothing.
+        """
+        from PIL import Image, ImageSequence
+
+        _ = errors
+        path = Path(file)
+        if self.engine is None:
+            self.engine = build_native_ocr_engine()
+        engine = self.engine
+        texts: List[str] = []
+        try:
+            with Image.open(path) as image:
+                frames = ImageSequence.Iterator(image) if path.suffix.lower() in (".tif", ".tiff") else [image]
+                for frame in frames:
+                    texts.append((engine.ocr_image(fit_to_pixel_budget(frame)) or "").strip())
+        except DocumentParseError:
+            raise
+        except Exception as exc:
+            raise DocumentParseError(f"Failed to OCR {path.name} ({engine.name}): {exc}") from exc
+
+        content = "\n\n".join(texts)
+        _check_near_empty(path.name, engine.name, content, max(1, len(texts)), len(texts))
+        self.last_engine = engine.name
+        self._last_metadata = {"parse_engine": engine.name, "ocr_pages": len(texts)}
+        return content
+
+    def get_file_metadata(self, file: Path) -> Dict:
+        """Engine and frame count for the most recently parsed file."""
+        _ = file
+        return dict(self._last_metadata)

docsgpt/parser/file/openapi3_parser.py

@@ -0,0 +1,51 @@
+from urllib.parse import urlparse
+
+from openapi_parser import parse
+
+try:
+    from docsgpt.parser.file.base_parser import BaseParser
+except ModuleNotFoundError:
+    from base_parser import BaseParser
+
+
+class OpenAPI3Parser(BaseParser):
+    def init_parser(self) -> None:
+        return super().init_parser()
+
+    def get_base_urls(self, urls):
+        base_urls = []
+        for i in urls:
+            parsed_url = urlparse(i)
+            base_url = parsed_url.scheme + "://" + parsed_url.netloc
+            if base_url not in base_urls:
+                base_urls.append(base_url)
+        return base_urls
+
+    def get_info_from_paths(self, path):
+        info = ""
+        if path.operations:
+            for operation in path.operations:
+                info += (
+                    f"\n{operation.method.value}="
+                    f"{operation.responses[0].description}"
+                )
+        return info
+
+    def parse_file(self, file_path):
+        data = parse(file_path)
+        results = ""
+        base_urls = self.get_base_urls(link.url for link in data.servers)
+        base_urls = ",".join([base_url for base_url in base_urls])
+        results += f"Base URL:{base_urls}\n"
+        i = 1
+        for path in data.paths:
+            info = self.get_info_from_paths(path)
+            results += (
+                f"Path{i}: {path.url}\n"
+                f"description: {path.description}\n"
+                f"parameters: {path.parameters}\nmethods: {info}\n"
+            )
+            i += 1
+        with open("results.txt", "w") as f:
+            f.write(results)
+        return results

docsgpt/parser/file/pdf_trust.py

@@ -0,0 +1,280 @@
+"""Trust checks for PDF text extraction.
+
+anydoc silently drops text from fonts it cannot map to Unicode — seen with
+non-embedded composite (Type0/CID) fonts relying on predefined CMaps, where
+the EN/ZH NDA corpus file loses its whole Chinese column with no error, and
+with Adobe-CNS1 fonts in HK legal PDFs where a valid ToUnicode exists but
+extraction still fails. These dependency-free checks scan the PDF's raw
+objects (including Flate-compressed object streams) so the pipeline can
+route such files to a heavier parser, or at least mark the output as
+unverified, instead of trusting silent partial text.
+
+Two stages:
+
+* ``check_pdf_fonts`` — pre-flight on the bytes alone: composite fonts
+  without an embedded ToUnicode CMap, and whether the PDF declares CJK font
+  resources at all.
+* ``verify_extraction`` — pre-flight plus the cross-check the font scan
+  cannot do: a PDF that declares CJK fonts whose extracted text contains
+  almost no CJK characters is not to be trusted.
+
+A flag means "verify or route to a fallback", not "the text is wrong": tools
+shipping Adobe's predefined CMaps (docling's parser does) often extract such
+files fine. On the benchmark corpus the checks caught both known
+silent-drop cases with zero false positives on the other 14 PDFs, and cost
+~30 ms per scanned MB (92 ms on a 3 MB, 150-page annual report).
+"""
+import re
+import zlib
+from pathlib import Path
+from typing import Dict, Iterator, List, Optional, Union
+
+_OBJ_HEADER = re.compile(rb"\d+\s+\d+\s+obj")
+_TYPE0 = re.compile(rb"/Subtype\s*/Type0")
+# The CJK expectation is keyed on CIDSystemInfo /Ordering ALONE — the
+# authoritative "this PDF maps text through a CJK character collection"
+# signal, present in every known silent-drop case. Matching CJK font *names*
+# (SimSun, MS-Gothic, MSungHK, ...) was tried and rejected: Word-exported
+# Latin PDFs routinely embed an MS-Gothic subset for a stray full-width
+# character (a 150-page English annual report in the benchmark corpus does),
+# and substring matching also catches Latin faces like FranklinGothic —
+# either way English-only documents would be rerouted to the heavy engine.
+_CJK_ORDERING = re.compile(rb"/Ordering\s*\((GB1|CNS1|Japan1|Japan2|KR|Korea1)\)")
+# Bytes kept around a Type0 marker found inside a decompressed object
+# stream: object streams hold many dicts with no obj/endobj markers, and a
+# font dict's keys sit close to its /Subtype entry.
+_WINDOW_BEFORE, _WINDOW_AFTER = 200, 800
+
+# Extracted text with fewer CJK characters than this, from a PDF that
+# declares CJK fonts, is treated as a silent drop.
+_MIN_CJK_CHARS = 10
+_CJK_PROBLEM_PREFIX = "PDF declares CJK fonts"
+
+_CJK_RANGES = (
+    ("一", "鿿"),  # CJK Unified Ideographs
+    ("぀", "ヿ"),  # Hiragana + Katakana
+    ("가", "힯"),  # Hangul syllables
+)
+
+
+# Per-stream decompression cap. Flate reaches ~1000:1, so uncapped inflation
+# of a crafted (or merely image-heavy) PDF inside the upload cap could balloon
+# to gigabytes and OOM the ingest worker. Font dicts and object streams — the
+# only things the signals live in — are far smaller than this.
+_STREAM_INFLATE_CAP = 4_000_000
+
+
+def _stream_bodies(raw: bytes) -> Iterator[bytes]:
+    """The raw bytes between each ``stream`` / ``endstream`` keyword pair.
+
+    A ``find`` walk rather than a regex: ``stream\\r?\\n(.*?)\\r?\\nendstream``
+    misses every stream whose data runs straight into ``endstream`` with no
+    EOL (common in the wild), and when it misses one its lazy group scans on
+    to the *next* stream's terminator — quadratic on large files (a 10 MB
+    file measured 32 s, matching a fifth of its streams) and a merged,
+    undecodable body for the rest.
+    """
+    pos = 0
+    while True:
+        pos = raw.find(b"stream", pos)
+        if pos < 0:
+            return
+        if raw[max(0, pos - 3):pos] == b"end":
+            pos += 6
+            continue
+        start = pos + 6
+        if raw[start:start + 2] == b"\r\n":
+            start += 2
+        elif raw[start:start + 1] in (b"\n", b"\r"):
+            start += 1
+        end = raw.find(b"endstream", start)
+        if end < 0:
+            return
+        body = raw[start:end]
+        if body.endswith(b"\r\n"):
+            body = body[:-2]
+        elif body.endswith((b"\n", b"\r")):
+            body = body[:-1]
+        pos = end + 9
+        yield body
+
+
+def _object_bodies(raw: bytes) -> Iterator[bytes]:
+    """The bytes between each ``N G obj`` header and the next ``endobj``.
+
+    A forward walk for the same reason as ``_stream_bodies``: the regex
+    ``\\d+\\s+\\d+\\s+obj(.*?)endobj`` rescans to end-of-file for every header
+    that has no ``endobj`` after it, so a malformed (or crafted) PDF with many
+    such headers costs headers x size — 2000 orphan headers in 1 MB measured
+    13.5 s, on the default upload path. Advancing past each ``endobj`` keeps
+    the walk linear and yields the same bodies.
+    """
+    pos = 0
+    while True:
+        header = _OBJ_HEADER.search(raw, pos)
+        if header is None:
+            return
+        end = raw.find(b"endobj", header.end())
+        if end < 0:
+            return
+        yield raw[header.end():end]
+        pos = end + 6
+
+
+def _decompressed_streams(raw: bytes) -> Iterator[bytes]:
+    """Each Flate stream in ``raw`` that decompresses, one at a time, capped."""
+    for body in _stream_bodies(raw):
+        try:
+            inflated = zlib.decompressobj().decompress(body, _STREAM_INFLATE_CAP)
+        except zlib.error:
+            continue
+        yield inflated
+
+
+def _cjk_chars(text: str, up_to: int) -> int:
+    """Count CJK characters in ``text``, stopping once ``up_to`` is reached."""
+    count = 0
+    for char in text:
+        for low, high in _CJK_RANGES:
+            if low <= char <= high:
+                count += 1
+                break
+        if count >= up_to:
+            break
+    return count
+
+
+def check_pdf_fonts(data: bytes) -> Dict[str, Union[int, bool]]:
+    """Pre-flight font scan of a PDF's bytes.
+
+    Args:
+        data: The complete PDF file contents.
+
+    Returns:
+        Dict with ``type0`` (composite fonts seen), ``type0_no_tounicode``
+        (those without an embedded ToUnicode CMap), ``expects_cjk`` (the PDF
+        declares CJK font resources), ``has_fonts``, and ``flagged`` — True
+        when extraction should not be trusted unverified.
+    """
+    type0 = bare = 0
+    expects_cjk = bool(_CJK_ORDERING.search(data))
+    has_fonts = b"/Font" in data
+
+    def count_font(unit: bytes) -> None:
+        """One analysis unit holding a Type0 font dict."""
+        nonlocal type0, bare
+        type0 += 1
+        if b"/ToUnicode" not in unit:
+            bare += 1
+
+    for body in _object_bodies(data):
+        if _TYPE0.search(body):
+            count_font(body)
+    # Each decompressed stream is scanned for every signal in one pass and
+    # then dropped; nothing inflated is retained past its loop iteration.
+    for stream in _decompressed_streams(data):
+        expects_cjk = expects_cjk or bool(_CJK_ORDERING.search(stream))
+        has_fonts = has_fonts or b"/Font" in stream
+        for type0_match in _TYPE0.finditer(stream):
+            start = type0_match.start()
+            count_font(stream[max(0, start - _WINDOW_BEFORE): start + _WINDOW_AFTER])
+    return {
+        "type0": type0,
+        "type0_no_tounicode": bare,
+        "expects_cjk": expects_cjk,
+        "has_fonts": has_fonts,
+        "flagged": bare > 0 or not has_fonts,
+    }
+
+
+def verify_extraction(data: bytes, markdown: str) -> List[str]:
+    """Reasons the extracted ``markdown`` of the PDF ``data`` shouldn't be trusted.
+
+    Combines the pre-flight font scan with the post-conversion cross-check it
+    cannot do alone: fonts with a valid ToUnicode that the converter
+    nevertheless failed to extract still show up as missing CJK output.
+
+    Args:
+        data: The complete PDF file contents.
+        markdown: The text a converter extracted from it.
+
+    Returns:
+        Human-readable problem strings; empty when the output looks sound.
+    """
+    problems: List[str] = []
+    result = check_pdf_fonts(data)
+    if result["type0_no_tounicode"]:
+        problems.append(
+            f"{result['type0_no_tounicode']} composite (Type0) font(s) carry no "
+            "ToUnicode map; extracted text may silently omit glyphs"
+        )
+    elif not result["has_fonts"]:
+        problems.append(
+            "no font resources detected; text extraction cannot be verified"
+        )
+    if result["expects_cjk"]:
+        cjk = _cjk_chars(markdown, _MIN_CJK_CHARS)
+        if cjk < _MIN_CJK_CHARS:
+            problems.append(
+                f"{_CJK_PROBLEM_PREFIX} but the extracted text contains only "
+                f"{cjk} CJK character(s)"
+            )
+    return problems
+
+
+def _text_layer_cjk_chars(file: Path, up_to: int) -> Optional[int]:
+    """CJK characters in the text layer as pypdfium2 reads it, stopping at ``up_to``.
+
+    pdfium ships Adobe's predefined CMaps, so it is an independent reference
+    for whether the document *has* CJK text to lose. None when pypdfium2 is
+    unavailable or the file cannot be read that way.
+    """
+    try:
+        import pypdfium2 as pdfium
+    except ImportError:
+        return None
+    try:
+        pdf = pdfium.PdfDocument(str(file))
+    except Exception:  # noqa: BLE001 - a reference probe, not a parse
+        return None
+    count = 0
+    try:
+        for index in range(len(pdf)):
+            page = pdf[index]
+            try:
+                textpage = page.get_textpage()
+                try:
+                    text = textpage.get_text_bounded()
+                finally:
+                    textpage.close()
+            finally:
+                page.close()
+            count += _cjk_chars(text, up_to - count)
+            if count >= up_to:
+                break
+    except Exception:  # noqa: BLE001
+        return None
+    finally:
+        pdf.close()
+    return count
+
+
+def verify_pdf_file(file: Path, markdown: str) -> List[str]:
+    """``verify_extraction`` for a file on disk; unreadable files trust the output.
+
+    The CJK cross-check is confirmed against pdfium's own text layer: a
+    stray ``/Ordering (Japan1)`` in an otherwise Latin document (a 48-page
+    Quartz export in the corpus carries one among 258 Identity fonts) is not
+    a dropped Chinese column, and must not send the file to a full docling
+    re-parse.
+    """
+    try:
+        data = Path(file).read_bytes()
+    except OSError:
+        return []
+    problems = verify_extraction(data, markdown)
+    if any(problem.startswith(_CJK_PROBLEM_PREFIX) for problem in problems):
+        reference = _text_layer_cjk_chars(Path(file), _MIN_CJK_CHARS)
+        if reference is not None and reference < _MIN_CJK_CHARS:
+            problems = [p for p in problems if not p.startswith(_CJK_PROBLEM_PREFIX)]
+    return problems

docsgpt/parser/file/pdfium_parser.py

@@ -0,0 +1,261 @@
+"""Direct PDF text-layer extraction with pypdfium2, for the attachment path.
+
+Attachments are read straight into a chat prompt; sources are chunked and
+embedded for retrieval. That difference is what justifies two parsers. Docling
+produces structured markdown — headings, table markup, image placeholders —
+which is worth its cost when the output feeds chunking, and mostly wasted when
+the output is concatenated into a prompt.
+
+The cost is not marginal. Measured on eight real uploads drawn from the p90 of
+production attachment-ingest latency (docling 2.119.0, OCR off, warm process):
+
+    docling total   155.6 s      pypdfium2 total   0.33 s      (~470x)
+    token yield within +/-15% per file, three files yielding *more* text
+
+What is lost is structure: docling emitted 20-187 headings and 44-88 table rows
+per file where this parser emits none, and text arrives in content-stream order
+rather than reading order. Acceptable for prompt context; not acceptable for
+retrieval, which is why `get_default_file_extractor` keeps docling by default
+and only swaps in this parser when a caller opts in.
+
+A scanned PDF has no text layer, so this parser would return an empty document
+for exactly the files that need docling most. It therefore probes first and
+delegates to `fallback_parser` when the text layer is too thin to be real,
+leaving scanned-PDF behaviour unchanged.
+"""
+
+import logging
+import statistics
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Union
+
+from docsgpt.parser.file.base_parser import BaseParser, DocumentParseError
+
+logger = logging.getLogger(__name__)
+
+# Pages sampled by the text-layer probe. Sampling a spread rather than the
+# first N matters: scanned documents routinely carry a text cover page, and
+# born-digital ones sometimes open with a full-page image.
+DEFAULT_SAMPLE_PAGES = 16
+
+# Median characters per sampled page below which the file is treated as having
+# no usable text layer. Measured separation on the production corpus was stark:
+# scanned files sat at 0 and 17 characters per page, text-layer files at
+# 433-6834, so anything in the low hundreds separates them. 32 is deliberately
+# conservative — it keeps sparse-but-real documents on docling.
+DEFAULT_MIN_MEDIAN_CHARS = 32
+
+
+class PdfiumTextParser(BaseParser):
+    """Extract a PDF's embedded text layer with pypdfium2.
+
+    Falls back to ``fallback_parser`` when the file has no usable text layer
+    (a scan), cannot be opened, or when pypdfium2 raises. The fallback is the
+    parser this one displaces, so delegating restores the previous behaviour
+    exactly rather than degrading it.
+
+    Attributes:
+        fallback_parser: Parser used when the text layer is unusable.
+        min_median_chars: Median chars/page required to use the fast path.
+        sample_pages: Maximum number of pages inspected by the probe.
+        last_engine: Name of the engine that produced the most recent parse,
+            ``"pypdfium2"`` or the fallback parser's class name.
+    """
+
+    def __init__(
+        self,
+        fallback_parser: Optional[BaseParser] = None,
+        min_median_chars: int = DEFAULT_MIN_MEDIAN_CHARS,
+        sample_pages: int = DEFAULT_SAMPLE_PAGES,
+        parser_config: Optional[Dict] = None,
+    ) -> None:
+        """Initialize the parser.
+
+        Args:
+            fallback_parser: Parser to delegate to when the text layer is
+                unusable. When ``None`` such files raise ``DocumentParseError``.
+            min_median_chars: Median characters per sampled page required to
+                take the fast path.
+            sample_pages: Maximum pages the probe inspects.
+            parser_config: Optional pre-built parser config.
+        """
+        super().__init__(parser_config)
+        self.fallback_parser = fallback_parser
+        self.min_median_chars = min_median_chars
+        self.sample_pages = max(1, sample_pages)
+        self.last_engine: Optional[str] = None
+        self._last_metadata: Dict[str, Any] = {}
+
+    def _init_parser(self) -> Dict:
+        """Initialize the parser config.
+
+        Returns:
+            Dict: Static config describing the probe thresholds.
+        """
+        return {
+            "engine": "pypdfium2",
+            "min_median_chars": self.min_median_chars,
+            "sample_pages": self.sample_pages,
+        }
+
+    def _sample_indices(self, page_count: int) -> List[int]:
+        """Return a spread of page indices to probe."""
+        if page_count <= self.sample_pages:
+            return list(range(page_count))
+        step = (page_count - 1) / (self.sample_pages - 1)
+        return sorted({round(i * step) for i in range(self.sample_pages)})
+
+    def _probe_text_layer(self, file: Path) -> Dict[str, Any]:
+        """Measure how much extractable text the PDF carries.
+
+        Args:
+            file: Path to the PDF.
+
+        Returns:
+            Dict: ``page_count`` and ``median_chars`` across sampled pages.
+
+        Raises:
+            Exception: Propagated from pypdfium2 when the file cannot be read.
+        """
+        import pypdfium2 as pdfium
+
+        pdf = pdfium.PdfDocument(str(file))
+        try:
+            page_count = len(pdf)
+            counts = []
+            for index in self._sample_indices(page_count):
+                page = pdf[index]
+                textpage = page.get_textpage()
+                try:
+                    counts.append(textpage.count_chars())
+                finally:
+                    textpage.close()
+                    page.close()
+        finally:
+            pdf.close()
+        return {
+            "page_count": page_count,
+            "median_chars": int(statistics.median(counts)) if counts else 0,
+        }
+
+    def _extract_text(self, file: Path) -> str:
+        """Extract every page's text layer.
+
+        Args:
+            file: Path to the PDF.
+
+        Returns:
+            str: Page texts joined by blank lines.
+        """
+        import pypdfium2 as pdfium
+
+        pdf = pdfium.PdfDocument(str(file))
+        pages: List[str] = []
+        try:
+            for index in range(len(pdf)):
+                page = pdf[index]
+                textpage = page.get_textpage()
+                try:
+                    pages.append(textpage.get_text_range())
+                finally:
+                    textpage.close()
+                    page.close()
+        finally:
+            pdf.close()
+        return "\n\n".join(pages)
+
+    def _delegate(self, file: Path, errors: str, reason: str) -> Union[str, List[str]]:
+        """Hand the file to the fallback parser.
+
+        Args:
+            file: Path to the PDF.
+            errors: Decoding leniency passed through to the fallback.
+            reason: Why the fast path was skipped, for the log line.
+
+        Returns:
+            The fallback parser's output.
+
+        Raises:
+            DocumentParseError: When no fallback parser is configured.
+        """
+        if self.fallback_parser is None:
+            raise DocumentParseError(
+                f"{file.name} has no usable PDF text layer ({reason}) and no "
+                "fallback parser is configured."
+            )
+        logger.info(
+            "PDF text-layer fast path skipped for %s (%s); using %s",
+            file.name,
+            reason,
+            type(self.fallback_parser).__name__,
+        )
+        if not self.fallback_parser.parser_config_set:
+            self.fallback_parser.init_parser()
+        self.last_engine = type(self.fallback_parser).__name__
+        result = self.fallback_parser.parse_file(file, errors=errors)
+        self._last_metadata = {
+            **self.fallback_parser.get_file_metadata(file),
+            "parse_engine": self.last_engine,
+        }
+        return result
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Parse a PDF, preferring its embedded text layer.
+
+        Args:
+            file: Path to the PDF.
+            errors: Decoding leniency, passed through to the fallback parser.
+
+        Returns:
+            The extracted text, or the fallback parser's output.
+
+        Raises:
+            DocumentParseError: When the text layer is unusable and no fallback
+                parser is configured.
+        """
+        file = Path(file)
+        try:
+            probe = self._probe_text_layer(file)
+        except Exception as exc:  # noqa: BLE001 - any read failure defers to docling
+            return self._delegate(file, errors, f"unreadable by pypdfium2: {exc}")
+
+        if probe["page_count"] == 0:
+            return self._delegate(file, errors, "no pages")
+        if probe["median_chars"] < self.min_median_chars:
+            return self._delegate(
+                file,
+                errors,
+                f"median {probe['median_chars']} chars/page below "
+                f"{self.min_median_chars}",
+            )
+
+        try:
+            text = self._extract_text(file)
+        except Exception as exc:  # noqa: BLE001 - as above
+            return self._delegate(file, errors, f"extraction failed: {exc}")
+
+        self.last_engine = "pypdfium2"
+        self._last_metadata = {
+            "parse_engine": "pypdfium2",
+            "pdf_pages": probe["page_count"],
+        }
+        logger.info(
+            "Parsed %s via pypdfium2 text layer: %d pages, %d chars",
+            file.name,
+            probe["page_count"],
+            len(text),
+        )
+        return text
+
+    def get_file_metadata(self, file: Path) -> Dict:
+        """Return metadata for the most recently parsed file.
+
+        Args:
+            file: Path to the PDF (unused; state is per-parse).
+
+        Returns:
+            Dict: Includes ``parse_engine`` so a stored attachment records
+            which parser actually produced its content.
+        """
+        _ = file
+        return dict(self._last_metadata)

docsgpt/parser/file/pptx_parser.py

@@ -0,0 +1,75 @@
+"""PPT parser.
+Contains parsers for presentation (.pptx) files to extract slide text.
+"""
+from pathlib import Path
+from typing import Any, Dict, List, Union
+
+from docsgpt.parser.file.base_parser import BaseParser
+
+class PPTXParser(BaseParser):
+    r"""PPTX (.pptx) parser for extracting text from PowerPoint slides.
+    Args:
+        concat_slides (bool): Specifies whether to concatenate all slide text into one document.
+            - If True, slide texts will be joined together as a single string.
+            - If False, each slide's text will be stored as a separate entry in a list.
+            Set to True by default.
+        slide_separator (str): Separator used to join slides' text content.
+            Only used when `concat_slides=True`. Default is "\n".
+        Refer to https://python-pptx.readthedocs.io/en/latest/ for more information.
+    """
+
+    def __init__(
+        self,
+        *args: Any,
+        concat_slides: bool = True,
+        slide_separator: str = "\n",
+        **kwargs: Any
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._concat_slides = concat_slides
+        self._slide_separator = slide_separator
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        r"""
+        Parse a .pptx file and extract text from each slide.
+        Args:
+            file (Path): Path to the .pptx file.
+            errors (str): Error handling policy ('ignore' by default).
+        Returns:
+            Union[str, List[str]]: Concatenated text if concat_slides is True,
+            otherwise a list of slide texts.
+        """
+
+        try:
+            from pptx import Presentation
+        except ImportError:
+            raise ImportError("pptx module is required to read .PPTX files.")
+
+        try:
+            presentation = Presentation(file)
+            slide_texts=[]
+
+            # Iterate over each slide in the presentation
+            for slide in presentation.slides:
+                slide_text=""
+
+                # Iterate over each shape in the slide
+                for shape in slide.shapes:
+                    # Check if the shape has a 'text' attribute and append that to the slide_text
+                    if hasattr(shape,"text"):
+                        slide_text+=shape.text
+
+                slide_texts.append(slide_text.strip())
+
+            if self._concat_slides:
+                return self._slide_separator.join(slide_texts)
+            else:
+                return slide_texts
+
+        except Exception as e:
+            raise e

docsgpt/parser/file/rst_parser.py

@@ -0,0 +1,201 @@
+"""reStructuredText parser.
+
+Contains parser for md files.
+
+"""
+import re
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+from docsgpt.parser.file.base_parser import BaseParser
+
+
+class RstParser(BaseParser):
+    """reStructuredText parser.
+
+    Extract text from .rst files.
+    Returns dictionary with keys as headers and values as the text between headers.
+
+    """
+
+    def __init__(
+            self,
+            *args: Any,
+            remove_hyperlinks: bool = True,
+            remove_images: bool = True,
+            remove_table_excess: bool = True,
+            remove_interpreters: bool = True,
+            remove_directives: bool = True,
+            remove_whitespaces_excess: bool = True,
+            # Be careful with remove_characters_excess, might cause data loss
+            remove_characters_excess: bool = True,
+            **kwargs: Any,
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._remove_hyperlinks = remove_hyperlinks
+        self._remove_images = remove_images
+        self._remove_table_excess = remove_table_excess
+        self._remove_interpreters = remove_interpreters
+        self._remove_directives = remove_directives
+        self._remove_whitespaces_excess = remove_whitespaces_excess
+        self._remove_characters_excess = remove_characters_excess
+
+    def rst_to_tups(self, rst_text: str) -> List[Tuple[Optional[str], str]]:
+        """Convert a reStructuredText file to a dictionary.
+
+        The keys are the headers and the values are the text under each header.
+
+        """
+        rst_tups: List[Tuple[Optional[str], str]] = []
+        lines = rst_text.split("\n")
+
+        current_header = None
+        current_text = ""
+
+        for i, line in enumerate(lines):
+            header_match = re.match(r"^[^\S\n]*[-=]+[^\S\n]*$", line)
+            if header_match and i > 0 and (
+                    len(lines[i - 1].strip()) == len(header_match.group().strip()) or lines[i - 2] == lines[i - 2]):
+                if current_header is not None:
+                    if current_text == "" or None:
+                        continue
+                    # removes the next heading from current Document
+                    if current_text.endswith(lines[i - 1] + "\n"):
+                        current_text = current_text[:len(current_text) - len(lines[i - 1] + "\n")]
+                    rst_tups.append((current_header, current_text))
+
+                current_header = lines[i - 1]
+                current_text = ""
+            else:
+                current_text += line + "\n"
+
+        rst_tups.append((current_header, current_text))
+
+        # TODO: Format for rst
+        #
+        # if current_header is not None:
+        #     # pass linting, assert keys are defined
+        #     rst_tups = [
+        #         (re.sub(r"#", "", cast(str, key)).strip(), re.sub(r"<.*?>", "", value))
+        #         for key, value in rst_tups
+        #     ]
+        # else:
+        #     rst_tups = [
+        #         (key, re.sub("\n", "", value)) for key, value in rst_tups
+        #     ]
+
+        if current_header is None:
+            rst_tups = [
+                (key, re.sub("\n", "", value)) for key, value in rst_tups
+            ]
+        return rst_tups
+
+    def chunk_by_token_count(self, text: str, max_tokens: int = 100) -> List[str]:
+        """Chunk text by token count."""
+    
+        avg_token_length = 5
+    
+        chunk_size = max_tokens * avg_token_length
+
+        chunks = []
+        for i in range(0, len(text), chunk_size):
+            chunk = text[i:i+chunk_size]
+            if i + chunk_size < len(text):
+                last_space = chunk.rfind(' ')
+                if last_space != -1:
+                    chunk = chunk[:last_space]
+            
+            chunks.append(chunk.strip())
+        
+        return chunks
+    
+    def remove_images(self, content: str) -> str:
+        pattern = r"\.\. image:: (.*)"
+        content = re.sub(pattern, "", content)
+        return content
+
+    def remove_hyperlinks(self, content: str) -> str:
+        pattern = r"`(.*?) <(.*?)>`_"
+        content = re.sub(pattern, r"\1", content)
+        return content
+
+    def remove_directives(self, content: str) -> str:
+        """Removes reStructuredText Directives"""
+        pattern = r"`\.\.([^:]+)::"
+        content = re.sub(pattern, "", content)
+        return content
+
+    def remove_interpreters(self, content: str) -> str:
+        """Removes reStructuredText Interpreted Text Roles"""
+        pattern = r":(\w+):"
+        content = re.sub(pattern, "", content)
+        return content
+
+    def remove_table_excess(self, content: str) -> str:
+        """Pattern to remove grid table separators"""
+        pattern = r"^\+[-]+\+[-]+\+$"
+        content = re.sub(pattern, "", content, flags=re.MULTILINE)
+        return content
+
+    def remove_whitespaces_excess(self, content: List[Tuple[str, Any]]) -> List[Tuple[str, Any]]:
+        """Pattern to match 2 or more consecutive whitespaces"""
+        pattern = r"\s{2,}"
+        content = [(key, re.sub(pattern, "  ", value)) for key, value in content]
+        return content
+
+    def remove_characters_excess(self, content: List[Tuple[str, Any]]) -> List[Tuple[str, Any]]:
+        """Pattern to match 2 or more consecutive characters"""
+        pattern = r"(\S)\1{2,}"
+        content = [(key, re.sub(pattern, r"\1\1\1", value, flags=re.MULTILINE)) for key, value in content]
+        return content
+
+    def _init_parser(self) -> Dict:
+        """Initialize the parser with the config."""
+        return {}
+
+    def parse_tups(
+            self, filepath: Path, errors: str = "ignore",max_tokens: Optional[int] = 1000
+    ) -> List[Tuple[Optional[str], str]]:
+        """Parse file into tuples."""
+        with open(filepath, "r") as f:
+            content = f.read()
+        if self._remove_hyperlinks:
+            content = self.remove_hyperlinks(content)
+        if self._remove_images:
+            content = self.remove_images(content)
+        if self._remove_table_excess:
+            content = self.remove_table_excess(content)
+        if self._remove_directives:
+            content = self.remove_directives(content)
+        if self._remove_interpreters:
+            content = self.remove_interpreters(content)
+        rst_tups = self.rst_to_tups(content)
+        if self._remove_whitespaces_excess:
+            rst_tups = self.remove_whitespaces_excess(rst_tups)
+        if self._remove_characters_excess:
+            rst_tups = self.remove_characters_excess(rst_tups)
+
+        # Apply chunking if max_tokens is provided
+        if max_tokens is not None:
+            chunked_tups = []
+            for header, text in rst_tups:
+                chunks = self.chunk_by_token_count(text, max_tokens)
+                for idx, chunk in enumerate(chunks):
+                    chunked_tups.append((f"{header} - Chunk {idx + 1}", chunk))
+            return chunked_tups    
+        return rst_tups
+
+    def parse_file(
+            self, filepath: Path, errors: str = "ignore"
+    ) -> Union[str, List[str]]:
+        """Parse file into string."""
+        tups = self.parse_tups(filepath, errors=errors)
+        results = []
+        # TODO: don't include headers right now
+        for header, value in tups:
+            if header is None:
+                results.append(value)
+            else:
+                results.append(f"\n\n{header}\n{value}")
+        return results

docsgpt/parser/file/tableize.py

@@ -0,0 +1,105 @@
+"""Reconstruct typographic tables in anydoc's PDF Markdown output.
+
+anydoc converts PDFs straight to Markdown with no document model, so tables
+drawn with dot leaders or bare whitespace alignment (financial statements,
+tables of contents) come out as flat text lines — values intact, structure
+lost. This post-processor detects runs of such lines and rewrites them as
+GFM tables: the lightweight alternative to a table-structure model for this
+layout family. (The robust fix is upstream in anydoc's Rust PDF code, where
+glyph x-positions exist; this is the recoverable-downstream version.)
+
+Deliberately conservative — a run converts only when it has at least
+``min_rows`` consecutive lines that each parse as ``label [leaders]
+numeric-columns`` with the *same* column count; anything else passes through
+untouched. Gated by ``ANYDOC_TABLEIZE`` (off by default) and applied only to
+anydoc's own PDF output, never to docling's.
+"""
+import re
+from typing import List, Optional, Tuple
+
+# label ...... 1,234 (56) — dot-leader row
+_LEADER = re.compile(r"^(.*?)\s*\.{3,}\s*(.+)$")
+# numeric-ish token: $ 1,234 / (30) / 83,431 / 12.5% / —
+_NUM_TOKEN = re.compile(r"\$?\(?-?\d[\d,.]*\)?%?|—")
+# row without leaders: label text, then a trailing run of numeric tokens
+_TRAILING = re.compile(r"^(.*?[^\s\d,.])\s+([\d$(].*)$")
+
+
+def _merge_currency(tokens: List[str]) -> List[str]:
+    """Join a free-standing ``$`` onto the number that follows it."""
+    out: List[str] = []
+    for token in tokens:
+        if out and out[-1] == "$":
+            out[-1] = "$" + token
+        else:
+            out.append(token)
+    return out
+
+
+def _parse_row(line: str) -> Optional[Tuple[str, List[str]]]:
+    """``(label, values)`` when the line looks like a typographic table row, else None.
+
+    Without dot leaders a row needs at least two numeric columns: a single
+    trailing number is what "Chapter 1", "Footnote 2", "ISO 9001" and
+    "Version 3.0" look like, and three of those in a row are a list, not a
+    table.
+    """
+    leader = _LEADER.match(line)
+    match = leader or _TRAILING.match(line)
+    if not match:
+        return None
+    label, rest = match.group(1).strip(" ."), match.group(2)
+    values = _merge_currency(rest.split())
+    if not values or not all(_NUM_TOKEN.fullmatch(v.lstrip("$")) for v in values):
+        return None
+    if leader is None and len(values) < 2:
+        return None
+    if not label or _NUM_TOKEN.fullmatch(label):
+        return None
+    return label, values
+
+
+def tableize(markdown: str, min_rows: int = 3) -> str:
+    """Rewrite runs of typographic table rows in ``markdown`` as GFM tables.
+
+    Args:
+        markdown: Converter output to post-process.
+        min_rows: Minimum consecutive, same-width rows for a run to convert.
+
+    Returns:
+        ``markdown`` with qualifying runs rewritten; everything else verbatim.
+    """
+    out: List[str] = []
+    run: List[Tuple[str, List[str], str]] = []  # (label, values, original line)
+
+    def flush() -> None:
+        nonlocal run
+        widths = {len(values) for _, values, _ in run}
+        if len(run) >= min_rows and len(widths) == 1:
+            ncols = widths.pop()
+            out.append("")
+            # GFM needs a header row; an empty one keeps invented labels out
+            # of the indexed text (they would otherwise be embedded as content).
+            out.append("|" + "   |" * (ncols + 1))
+            out.append("|" + " --- |" * (ncols + 1))
+            for label, values, _ in run:
+                # Only the label can carry a '|' (values are numeric tokens);
+                # unescaped it would add a cell and mis-column the row.
+                cells = [label.replace("|", "\\|")] + values
+                out.append("| " + " | ".join(cells) + " |")
+            out.append("")
+        else:
+            out.extend(original for _, _, original in run)
+        run = []
+
+    for line in markdown.splitlines():
+        parsed = _parse_row(line.strip()) if line.strip() else None
+        if parsed:
+            run.append((*parsed, line))
+        else:
+            if run:
+                flush()
+            out.append(line)
+    if run:
+        flush()
+    return "\n".join(out)

docsgpt/parser/file/tabular_parser.py

@@ -0,0 +1,280 @@
+"""Tabular parser.
+
+Contains parsers for tabular data files.
+
+"""
+import datetime
+import math
+from pathlib import Path
+from typing import Any, Dict, List, Union
+
+from docsgpt.parser.file.base_parser import BaseParser
+
+
+def cell_to_text(value: Any) -> str:
+    """Render one spreadsheet/CSV cell as text that is always safe to join.
+
+    ``Series.astype(str)`` cannot be trusted for this. Up to pandas 2.x it
+    rendered missing values as the string ``"nan"``; pandas 3.0 *preserves*
+    them, so ``.tolist()`` hands back Python ``float`` NaNs and ``str.join``
+    raises ``TypeError: sequence item N: expected str instance, float found``.
+    A single blank cell anywhere in a sheet was therefore enough to destroy
+    an entire upload.
+
+    Missing values (blank cells, ``NaT``, Excel error values, and formulas
+    with no cached result) become the empty string. Dates and times are
+    rendered ISO-8601 rather than pandas' repr. Integral floats drop the
+    ``.0`` that pandas adds when a blank upcasts an integer column — an ID
+    column with one gap otherwise turns every id into ``1001.0``. NUL bytes
+    are stripped here because they crash the downstream Postgres text write.
+
+    Args:
+        value: A single cell value, of any type pandas or openpyxl may yield.
+
+    Returns:
+        str: The cell rendered as text; ``""`` for missing values.
+    """
+    if value is None:
+        return ""
+    # Handled before the pandas lookup below: it is by far the common case
+    # (every blank cell) and needs no import.
+    if isinstance(value, float) and math.isnan(value):
+        return ""
+    try:
+        import pandas as pd
+    except ImportError:
+        # pandas is optional here — plain float NaN is already covered above,
+        # and pd.NA/NaT can only reach us via pandas in the first place.
+        pd = None
+    if pd is not None:
+        try:
+            if pd.isna(value):
+                return ""
+        except (TypeError, ValueError):
+            # pd.isna raises on array-like cells (lists, ndarrays). Those are
+            # real values rather than missing data, so fall through and
+            # stringify them below.
+            pass
+    if isinstance(value, (datetime.datetime, datetime.date, datetime.time)):
+        return value.isoformat()
+    if isinstance(value, float) and value.is_integer():
+        return str(int(value))
+    text = str(value)
+    return text.replace("\x00", "") if "\x00" in text else text
+
+
+def _row_to_texts(row: Any) -> List[str]:
+    """Render a pandas row as a list of join-safe strings."""
+    return [cell_to_text(v) for v in row.tolist()]
+
+
+class CSVParser(BaseParser):
+    """CSV parser.
+
+    Args:
+        concat_rows (bool): whether to concatenate all rows into one document.
+            If set to False, a Document will be created for each row.
+            True by default.
+
+    """
+
+    def __init__(self, *args: Any, concat_rows: bool = True, **kwargs: Any) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._concat_rows = concat_rows
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Parse file.
+
+        Returns:
+            Union[str, List[str]]: a string or a List of strings.
+
+        """
+        try:
+            import csv
+        except ImportError:
+            raise ValueError("csv module is required to read CSV files.")
+        text_list = []
+        with open(file, "r") as fp:
+            csv_reader = csv.reader(fp)
+            for row in csv_reader:
+                text_list.append(", ".join(row))
+        if self._concat_rows:
+            return "\n".join(text_list)
+        else:
+            return text_list
+
+
+class PandasCSVParser(BaseParser):
+    r"""Pandas-based CSV parser.
+
+    Parses CSVs using the separator detection from Pandas `read_csv`function.
+    If special parameters are required, use the `pandas_config` dict.
+
+    Args:
+        concat_rows (bool): whether to concatenate all rows into one document.
+            If set to False, a Document will be created for each row.
+            True by default.
+
+        col_joiner (str): Separator to use for joining cols per row.
+            Set to ", " by default.
+
+        row_joiner (str): Separator to use for joining each row.
+            Only used when `concat_rows=True`.
+            Set to "\n" by default.
+
+        pandas_config (dict): Options for the `pandas.read_csv` function call.
+            Refer to https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html
+            for more information.
+            Set to empty dict by default, this means pandas will try to figure
+            out the separators, table head, etc. on its own.
+            
+        header_period (int): Controls how headers are included in output:
+            - 0: Headers only at the beginning
+            - 1: Headers in every row
+            - N > 1: Headers every N rows
+            
+        header_prefix (str): Prefix for header rows. Default is "HEADERS: ".
+    """
+
+    def __init__(
+            self,
+            *args: Any,
+            concat_rows: bool = True,
+            col_joiner: str = ", ",
+            row_joiner: str = "\n",
+            pandas_config: dict = {},
+            header_period: int = 20,
+            header_prefix: str = "HEADERS: ",
+            **kwargs: Any
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._concat_rows = concat_rows
+        self._col_joiner = col_joiner
+        self._row_joiner = row_joiner
+        self._pandas_config = pandas_config
+        self._header_period = header_period
+        self._header_prefix = header_prefix
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Parse file."""
+        try:
+            import pandas as pd
+        except ImportError:
+            raise ValueError("pandas module is required to read CSV files.")
+
+        df = pd.read_csv(file, **self._pandas_config)
+        headers = [cell_to_text(h) for h in df.columns.tolist()]
+        header_row = f"{self._header_prefix}{self._col_joiner.join(headers)}"
+
+        if not self._concat_rows:
+            return df.apply(
+                lambda row: (self._col_joiner).join(_row_to_texts(row)), axis=1
+            ).tolist()
+
+        text_list = []
+        if self._header_period != 1:
+            text_list.append(header_row)
+
+        for i, row in df.iterrows():
+            if (self._header_period > 1 and i > 0 and i % self._header_period == 0):
+                text_list.append(header_row)
+            text_list.append(self._col_joiner.join(_row_to_texts(row)))
+            if self._header_period == 1 and i < len(df) - 1:
+                text_list.append(header_row)
+
+        return self._row_joiner.join(text_list)
+
+
+class ExcelParser(BaseParser):
+    r"""Excel (.xlsx) parser.
+
+    Parses Excel files using Pandas `read_excel` function.
+    If special parameters are required, use the `pandas_config` dict.
+
+    Args:
+        concat_rows (bool): whether to concatenate all rows into one document.
+            If set to False, a Document will be created for each row.
+            True by default.
+
+        col_joiner (str): Separator to use for joining cols per row.
+            Set to ", " by default.
+
+        row_joiner (str): Separator to use for joining each row.
+            Only used when `concat_rows=True`.
+            Set to "\n" by default.
+
+        pandas_config (dict): Options for the `pandas.read_excel` function call.
+            Refer to https://pandas.pydata.org/docs/reference/api/pandas.read_excel.html
+            for more information.
+            Set to empty dict by default, this means pandas will try to figure
+            out the table structure on its own.
+            
+        header_period (int): Controls how headers are included in output:
+            - 0: Headers only at the beginning (default)
+            - 1: Headers in every row
+            - N > 1: Headers every N rows
+            
+        header_prefix (str): Prefix for header rows. Default is "HEADERS: ".
+    """
+
+    def __init__(
+            self,
+            *args: Any,
+            concat_rows: bool = True,
+            col_joiner: str = ", ",
+            row_joiner: str = "\n",
+            pandas_config: dict = {},
+            header_period: int = 20,
+            header_prefix: str = "HEADERS: ",
+            **kwargs: Any
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._concat_rows = concat_rows
+        self._col_joiner = col_joiner
+        self._row_joiner = row_joiner
+        self._pandas_config = pandas_config
+        self._header_period = header_period
+        self._header_prefix = header_prefix
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Parse file."""
+        try:
+            import pandas as pd
+        except ImportError:
+            raise ValueError("pandas module is required to read Excel files.")
+
+        df = pd.read_excel(file, **self._pandas_config)
+        headers = [cell_to_text(h) for h in df.columns.tolist()]
+        header_row = f"{self._header_prefix}{self._col_joiner.join(headers)}"
+
+        if not self._concat_rows:
+            return df.apply(
+                lambda row: (self._col_joiner).join(_row_to_texts(row)), axis=1
+            ).tolist()
+
+        text_list = []
+        if self._header_period != 1:
+            text_list.append(header_row)
+
+        for i, row in df.iterrows():
+            if (self._header_period > 1 and i > 0 and i % self._header_period == 0):
+                text_list.append(header_row)
+            text_list.append(self._col_joiner.join(_row_to_texts(row)))
+            if self._header_period == 1 and i < len(df) - 1:
+                text_list.append(header_row)
+        return self._row_joiner.join(text_list)

docsgpt/parser/remote/base.py

@@ -0,0 +1,19 @@
+"""Base reader class."""
+from abc import abstractmethod
+from typing import Any, List
+
+from docsgpt.parser.schema.base import Document
+from docsgpt.vectorstore.document_class import Document as VectorDocument
+
+
+class BaseRemote:
+    """Utilities for loading data from a directory."""
+
+    @abstractmethod
+    def load_data(self, *args: Any, **load_kwargs: Any) -> List[Document]:
+        """Load data from the input directory."""
+
+    def load_vector_documents(self, **load_kwargs: Any) -> List[VectorDocument]:
+        """Load data in the vector-store document format."""
+        docs = self.load_data(**load_kwargs)
+        return [d.to_vector_format() for d in docs]

docsgpt/parser/remote/crawler_loader.py

@@ -0,0 +1,102 @@
+import logging
+import os
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin, urlparse
+
+from docsgpt.parser.remote.base import BaseRemote
+from docsgpt.parser.schema.base import Document
+from docsgpt.core.url_validation import validate_url, SSRFError
+from docsgpt.security.safe_url import pinned_request
+
+
+class CrawlerLoader(BaseRemote):
+    def __init__(self, limit=10):
+        self.limit = limit  # Set the limit for the number of pages to scrape
+
+    def load_data(self, inputs):
+        url = inputs
+        if isinstance(url, list) and url:
+            url = url[0]
+
+        # Validate URL to prevent SSRF attacks
+        try:
+            url = validate_url(url)
+        except SSRFError as e:
+            logging.error(f"URL validation failed: {e}")
+            return []
+
+        visited_urls = set()
+        base_url = urlparse(url).scheme + "://" + urlparse(url).hostname
+        urls_to_visit = [url]
+        loaded_content = []
+
+        while urls_to_visit:
+            current_url = urls_to_visit.pop(0)
+            visited_urls.add(current_url)
+
+            try:
+                response = pinned_request("GET", current_url, timeout=30)
+                response.raise_for_status()
+                soup = BeautifulSoup(response.text, "html.parser")
+                extra_info = {
+                    "source": current_url,
+                    "file_path": self._url_to_virtual_path(current_url),
+                }
+                # Mirror WebLoader: without a title, citations fall back to the
+                # chunk body and render as a wall of text.
+                if soup.title:
+                    title = soup.title.get_text(strip=True)
+                    if title:
+                        extra_info["title"] = title
+                loaded_content.append(
+                    Document(
+                        soup.get_text(separator="\n", strip=True),
+                        extra_info=extra_info,
+                    )
+                )
+            except Exception as e:
+                logging.error(f"Error processing URL {current_url}: {e}", exc_info=True)
+                continue
+
+            # Parse the HTML content to extract all links
+            all_links = [
+                urljoin(current_url, a['href'])
+                for a in soup.find_all('a', href=True)
+                if base_url in urljoin(current_url, a['href'])
+            ]
+
+            # Add new links to the list of URLs to visit if they haven't been visited yet
+            urls_to_visit.extend([link for link in all_links if link not in visited_urls])
+            urls_to_visit = list(set(urls_to_visit))
+
+            # Stop crawling if the limit of pages to scrape is reached
+            if self.limit is not None and len(visited_urls) >= self.limit:
+                break
+
+        return loaded_content
+
+    def _url_to_virtual_path(self, url):
+        """
+        Convert a URL to a virtual file path ending with .md.
+
+        Examples:
+            https://docs.docsgpt.cloud/ -> index.md
+            https://docs.docsgpt.cloud/guides/setup -> guides/setup.md
+            https://docs.docsgpt.cloud/guides/setup/ -> guides/setup.md
+            https://example.com/page.html -> page.md
+        """
+        parsed = urlparse(url)
+        path = parsed.path.strip("/")
+
+        if not path:
+            return "index.md"
+
+        # Remove common file extensions and add .md
+        base, ext = os.path.splitext(path)
+        if ext.lower() in [".html", ".htm", ".php", ".asp", ".aspx", ".jsp"]:
+            path = base
+
+        if not path.endswith(".md"):
+            path = f"{path}.md"
+
+        return path

docsgpt/parser/remote/crawler_markdown.py

@@ -0,0 +1,187 @@
+from urllib.parse import urlparse, urljoin
+from bs4 import BeautifulSoup
+from docsgpt.parser.remote.base import BaseRemote
+from docsgpt.core.url_validation import validate_url, SSRFError
+from docsgpt.security.safe_url import UnsafeUserUrlError, pinned_request
+import re
+from markdownify import markdownify
+from docsgpt.parser.schema.base import Document
+import tldextract
+import os
+
+# The bundled public-suffix snapshot is enough for domain matching; the
+# default extractor would fetch the live list on first use and cache it on
+# disk, which is a network round trip the ingest worker should not depend on.
+_extract = tldextract.TLDExtract(suffix_list_urls=(), cache_dir=None)
+
+class CrawlerLoader(BaseRemote):
+    def __init__(self, limit=10, allow_subdomains=False):
+        """
+        Given a URL crawl web pages up to `self.limit`,
+        convert HTML content to Markdown, and returning a list of Document objects.
+
+        :param limit: The maximum number of pages to crawl.
+        :param allow_subdomains: If True, crawl pages on subdomains of the base domain.
+        """
+        self.limit = limit
+        self.allow_subdomains = allow_subdomains
+
+    def load_data(self, inputs):
+        url = inputs
+        if isinstance(url, list) and url:
+            url = url[0]
+
+        # Validate URL to prevent SSRF attacks
+        try:
+            url = validate_url(url)
+        except SSRFError as e:
+            print(f"URL validation failed: {e}")
+            return []
+
+        # Keep track of visited URLs to avoid revisiting the same page
+        visited_urls = set()
+
+        # Determine the base domain for link filtering using tldextract
+        base_domain = self._get_base_domain(url)
+        urls_to_visit = {url}
+        documents = []
+
+        while urls_to_visit:
+            current_url = urls_to_visit.pop()
+
+            # Skip if already visited
+            if current_url in visited_urls:
+                continue
+            visited_urls.add(current_url)
+
+            # Fetch the page content
+            html_content = self._fetch_page(current_url)
+            if html_content is None:
+                continue
+
+            # Convert the HTML to Markdown for cleaner text formatting
+            title, language, processed_markdown = self._process_html_to_markdown(html_content, current_url)
+            if processed_markdown:
+                # Generate virtual file path from URL for consistent file-like matching
+                virtual_path = self._url_to_virtual_path(current_url)
+                
+                # Create a Document for each visited page
+                documents.append(
+                    Document(
+                        processed_markdown,  # content
+                        None,  # doc_id
+                        None,  # embedding
+                        {
+                            "source": current_url,
+                            "title": title,
+                            "language": language,
+                            "file_path": virtual_path,
+                        },  # extra_info
+                    )
+                )
+
+            # Extract links and filter them according to domain rules
+            new_links = self._extract_links(html_content, current_url)
+            filtered_links = self._filter_links(new_links, base_domain)
+
+            # Add any new, not-yet-visited links to the queue
+            urls_to_visit.update(link for link in filtered_links if link not in visited_urls)
+
+            # If we've reached the limit, stop crawling
+            if self.limit is not None and len(visited_urls) >= self.limit:
+                break
+
+        return documents
+
+    def _fetch_page(self, url):
+        try:
+            response = pinned_request("GET", url, timeout=10)
+            response.raise_for_status()
+            return response.text
+        except UnsafeUserUrlError as e:
+            print(f"URL validation failed for {url}: {e}")
+            return None
+        except Exception as e:
+            print(f"Error fetching URL {url}: {e}")
+            return None
+
+    def _process_html_to_markdown(self, html_content, current_url):
+        soup = BeautifulSoup(html_content, 'html.parser')
+        title_tag = soup.find('title')
+        title = title_tag.text.strip() if title_tag else "No Title"
+
+        # Extract language
+        language_tag = soup.find('html')
+        language = language_tag.get('lang', 'en') if language_tag else "en"
+
+        markdownified = markdownify(html_content, heading_style="ATX", newline_style="BACKSLASH")
+        # Collapse runs of blank lines to a single one — the same shape
+        # ``html_to_markdown`` gives uploaded HTML files.
+        markdownified = re.sub(r'\n{3,}', '\n\n', markdownified)
+        return title, language, markdownified
+
+    def _extract_links(self, html_content, current_url):
+        soup = BeautifulSoup(html_content, 'html.parser')
+        links = []
+        for a in soup.find_all('a', href=True):
+            full_url = urljoin(current_url, a['href'])
+            links.append((full_url, a.text.strip()))
+        return links
+
+    def _get_base_domain(self, url):
+        extracted = _extract(url)
+        # Reconstruct the domain as domain.suffix
+        base_domain = f"{extracted.domain}.{extracted.suffix}"
+        return base_domain
+
+    def _filter_links(self, links, base_domain):
+        """
+        Filter the extracted links to only include those that match the crawling criteria:
+        - If allow_subdomains is True, allow any link whose domain ends with the base_domain.
+        - If allow_subdomains is False, only allow exact matches of the base_domain.
+        """
+        filtered = []
+        for link, _ in links:
+            parsed_link = urlparse(link)
+            if not parsed_link.netloc:
+                continue
+
+            extracted = _extract(parsed_link.netloc)
+            link_base = f"{extracted.domain}.{extracted.suffix}"
+
+            if self.allow_subdomains:
+                # For subdomains: sub.example.com ends with example.com
+                if link_base == base_domain or link_base.endswith("." + base_domain):
+                    filtered.append(link)
+            else:
+                # Exact domain match
+                if link_base == base_domain:
+                    filtered.append(link)
+        return filtered
+
+    def _url_to_virtual_path(self, url):
+        """
+        Convert a URL to a virtual file path ending with .md.
+
+        Examples:
+            https://docs.docsgpt.cloud/ -> index.md
+            https://docs.docsgpt.cloud/guides/setup -> guides/setup.md
+            https://docs.docsgpt.cloud/guides/setup/ -> guides/setup.md
+            https://example.com/page.html -> page.md
+        """
+        parsed = urlparse(url)
+        path = parsed.path.strip("/")
+
+        if not path:
+            return "index.md"
+
+        # Remove common file extensions and add .md
+        base, ext = os.path.splitext(path)
+        if ext.lower() in [".html", ".htm", ".php", ".asp", ".aspx", ".jsp"]:
+            path = base
+
+        # Ensure path ends with .md
+        if not path.endswith(".md"):
+            path = path + ".md"
+
+        return path

docsgpt/parser/remote/github_loader.py

@@ -0,0 +1,454 @@
+import base64
+import logging
+import mimetypes
+import time
+from concurrent.futures import ThreadPoolExecutor
+from typing import Dict, List, Optional, Tuple
+
+import requests
+
+from docsgpt.core.settings import settings
+from docsgpt.parser.remote.base import BaseRemote
+from docsgpt.parser.schema.base import Document
+
+logger = logging.getLogger(__name__)
+
+# Directory names that hold vendored or generated output. Anything under one
+# of these is build product, not source: it bloats the index, costs an API
+# call each, and answers no question a user would ask of the repo.
+SKIP_DIRECTORIES = {
+    ".git", ".github/workflows/generated", ".idea", ".vscode",
+    "__pycache__", ".mypy_cache", ".pytest_cache", ".tox", ".venv", "venv",
+    "node_modules", "bower_components", "vendor", "third_party", "external",
+    "dist", "build", "out", "target", "bin", "obj",
+    "site-packages", "coverage", "htmlcov", ".next", ".nuxt", ".svelte-kit",
+}
+
+# Exact filenames that are machine-generated and semantically empty.
+SKIP_FILENAMES = {
+    "package-lock.json", "yarn.lock", "pnpm-lock.yaml", "bun.lockb",
+    "composer.lock", "gemfile.lock", "cargo.lock", "poetry.lock",
+    "pipfile.lock", "go.sum", "mix.lock", "podfile.lock",
+}
+
+# Suffixes that mark minified or map output even under a source directory.
+SKIP_SUFFIXES = (
+    ".min.js", ".min.css", ".map", ".lock",
+    ".pyc", ".pyo", ".class", ".jar", ".war", ".o", ".so", ".dylib", ".dll",
+    ".exe", ".wasm", ".bin", ".pdb",
+)
+
+
+class GitHubLoader(BaseRemote):
+    """Load a GitHub repository's text files as ``Document`` objects.
+
+    Uses the git *tree* API to enumerate the repository in a single request
+    rather than walking ``/contents/`` directory by directory, filters out
+    binaries, vendored output and oversized blobs *before* fetching them, and
+    downloads the survivors in parallel.
+    """
+
+    def __init__(self):
+        self.access_token = settings.GITHUB_ACCESS_TOKEN
+        self.headers = {
+            "Authorization": f"token {self.access_token}",
+            "Accept": "application/vnd.github.v3+json"
+        } if self.access_token else {
+            "Accept": "application/vnd.github.v3+json"
+        }
+        return
+
+    def is_text_file(self, file_path: str) -> bool:
+        """Determine if a file is a text file based on extension."""
+        # Common text file extensions
+        text_extensions = {
+            '.txt', '.md', '.markdown', '.rst', '.adoc', '.org',
+            '.json', '.jsonc', '.xml', '.yaml', '.yml', '.toml',
+            '.py', '.pyi', '.js', '.mjs', '.cjs', '.ts', '.tsx', '.jsx',
+            '.vue', '.svelte', '.astro',
+            '.java', '.c', '.cc', '.cpp', '.h', '.hpp', '.hxx',
+            '.cs', '.go', '.rs', '.rb', '.php', '.swift', '.kt', '.kts',
+            '.scala', '.sc', '.clj', '.cljs', '.edn', '.ex', '.exs', '.erl',
+            '.hs', '.ml', '.mli', '.fs', '.fsx', '.dart', '.lua', '.jl',
+            '.zig', '.nim', '.v', '.pl', '.pm', '.groovy', '.gradle',
+            '.html', '.htm', '.css', '.scss', '.sass', '.less',
+            '.sh', '.bash', '.zsh', '.fish', '.ps1', '.bat',
+            '.sql', '.graphql', '.gql', '.proto', '.thrift',
+            '.tf', '.tfvars', '.hcl', '.dockerfile', '.cmake', '.mk',
+            '.ini', '.cfg', '.conf', '.config', '.properties',
+            '.gitignore', '.dockerignore', '.editorconfig', '.gitattributes',
+            '.csv', '.tsv',
+        }
+
+        # Get file extension
+        file_lower = file_path.lower()
+        for ext in text_extensions:
+            if file_lower.endswith(ext):
+                return True
+
+        # Extension-less files that are conventionally text.
+        basename = file_lower.rsplit("/", 1)[-1]
+        if basename in {
+            "dockerfile", "makefile", "rakefile", "gemfile", "procfile",
+            "license", "licence", "notice", "authors", "codeowners", "readme",
+        }:
+            return True
+
+        # Also check MIME type
+        mime_type, _ = mimetypes.guess_type(file_path)
+        if mime_type and (mime_type.startswith("text") or mime_type in ["application/json", "application/xml"]):
+            return True
+
+        return False
+
+    def should_skip_path(self, file_path: str) -> bool:
+        """Return ``True`` for vendored, generated or binary-by-name paths.
+
+        Applied to the tree listing before any content is fetched, so a
+        skipped file costs nothing. Complements :meth:`is_text_file`, which
+        only knows about extensions.
+
+        Args:
+            file_path: Repo-relative path, e.g. ``"web/dist/app.min.js"``.
+
+        Returns:
+            ``True`` when the path should not be ingested.
+        """
+        lowered = file_path.lower()
+        parts = lowered.split("/")
+        if any(part in SKIP_DIRECTORIES for part in parts[:-1]):
+            return True
+        if parts[-1] in SKIP_FILENAMES:
+            return True
+        if lowered.endswith(SKIP_SUFFIXES):
+            return True
+        # Dot-directories (.git, .cache, ...) never carry documentation.
+        if any(part.startswith(".") and part not in {".github"} for part in parts[:-1]):
+            return True
+        return False
+
+    def _max_file_bytes(self) -> int:
+        """Resolve the per-blob size cap; ``0`` disables it."""
+        raw = getattr(settings, "GITHUB_INGEST_MAX_FILE_BYTES", None)
+        if isinstance(raw, bool) or not isinstance(raw, (int, str)):
+            return 1048576
+        try:
+            return max(0, int(raw))
+        except (TypeError, ValueError):
+            return 1048576
+
+    def _max_workers(self) -> int:
+        """Resolve the parallel-fetch width, clamped to a sane range."""
+        raw = getattr(settings, "GITHUB_INGEST_MAX_WORKERS", None)
+        if isinstance(raw, bool) or not isinstance(raw, (int, str)):
+            return 8
+        try:
+            return max(1, min(32, int(raw)))
+        except (TypeError, ValueError):
+            return 8
+
+    @staticmethod
+    def normalize_repo(repo_url: str) -> str:
+        """Reduce a user-pasted repo URL to ``owner/name``.
+
+        Strips the scheme/host, a trailing ``.git`` and any trailing slash —
+        the three shapes that previously 404'd the contents API.
+
+        Args:
+            repo_url: Anything from ``owner/name`` to
+                ``https://github.com/owner/name.git/``.
+
+        Returns:
+            The ``owner/name`` segment.
+        """
+        repo = (repo_url or "").strip()
+        if not repo:
+            return ""
+
+        if repo.startswith("git@"):
+            # git@github.com:owner/name.git
+            host, _, path = repo[len("git@"):].partition(":")
+            if host.lower() != "github.com":
+                return ""
+            repo = path
+        elif "://" in repo:
+            remainder = repo.split("://", 1)[1]
+            host, _, path = remainder.partition("/")
+            # Anything not hosted on github.com is not a repo. Previously such
+            # a URL was pasted straight into the contents path and 404-looped.
+            if host.lower() not in {"github.com", "www.github.com"}:
+                return ""
+            repo = path
+        elif repo.lower().startswith(("github.com/", "www.github.com/")):
+            repo = repo.split("/", 1)[1]
+
+        repo = repo.strip("/")
+        if repo.lower().endswith(".git"):
+            repo = repo[: -len(".git")]
+        parts = [p for p in repo.split("/") if p]
+        if len(parts) < 2:
+            return ""
+        owner, name = parts[0], parts[1]
+        if name.lower().endswith(".git"):
+            name = name[: -len(".git")]
+        # A bare host slipping through ("suat-handbook.netlify.app") has no
+        # owner segment, so require both halves to look like path segments.
+        if not owner or not name or ":" in owner or ":" in name:
+            return ""
+        return f"{owner}/{name}"
+
+    def get_default_branch(self, repo_name: str) -> str:
+        """Return the repo's default branch, falling back to ``main``.
+
+        The blob URL used for citations was hard-coded to ``main``, which
+        produced dead source links for every ``master``-default repo.
+        """
+        try:
+            response = self._make_request(f"https://api.github.com/repos/{repo_name}")
+            branch = response.json().get("default_branch")
+            if branch:
+                return str(branch)
+        except Exception as e:
+            logger.warning(
+                "Could not resolve default branch for %s (%s); assuming 'main'",
+                repo_name, e,
+            )
+        return "main"
+
+    def fetch_repo_tree(
+        self, repo_name: str, branch: str
+    ) -> Tuple[List[Tuple[str, int]], bool]:
+        """List every blob in the repo with one recursive tree request.
+
+        Replaces the per-directory ``/contents/`` walk, which cost one API
+        call per directory (720 for a mid-size repo) before a single file
+        was read.
+
+        Args:
+            repo_name: ``owner/name``.
+            branch: Branch or ref to enumerate.
+
+        Returns:
+            ``(entries, truncated)`` where ``entries`` is a list of
+            ``(path, size_bytes)`` and ``truncated`` flags a repo too large
+            for the tree endpoint (caller should fall back to the walk).
+        """
+        url = (
+            f"https://api.github.com/repos/{repo_name}/git/trees/"
+            f"{branch}?recursive=1"
+        )
+        response = self._make_request(url)
+        payload = response.json()
+        if isinstance(payload, dict) and "tree" not in payload and "message" in payload:
+            raise Exception(f"GitHub API error: {payload.get('message')}")
+        entries = [
+            (item.get("path", ""), int(item.get("size") or 0))
+            for item in payload.get("tree", [])
+            if item.get("type") == "blob"
+        ]
+        return entries, bool(payload.get("truncated"))
+
+    def select_files(self, entries: List[Tuple[str, int]]) -> List[str]:
+        """Filter tree entries down to the paths worth fetching.
+
+        Args:
+            entries: ``(path, size_bytes)`` pairs from :meth:`fetch_repo_tree`.
+
+        Returns:
+            Repo-relative paths that pass the skip-list, the text-extension
+            check and the size cap, in tree order.
+        """
+        max_bytes = self._max_file_bytes()
+        selected: List[str] = []
+        skipped_big = 0
+        for path, size in entries:
+            if not path or self.should_skip_path(path) or not self.is_text_file(path):
+                continue
+            if max_bytes and size > max_bytes:
+                skipped_big += 1
+                continue
+            selected.append(path)
+        if skipped_big:
+            logger.info(
+                "Skipped %d file(s) over the %d-byte cap", skipped_big, max_bytes
+            )
+        return selected
+
+    def fetch_file_content(self, repo_url: str, file_path: str) -> Optional[str]:
+        """Fetch file content. Returns None if file should be skipped (binary files or empty files)."""
+        url = f"https://api.github.com/repos/{repo_url}/contents/{file_path}"
+        response = self._make_request(url)
+
+        content = response.json()
+
+        if content.get("encoding") == "base64":
+            if self.is_text_file(file_path):  # Handle only text files
+                try:
+                    decoded_content = base64.b64decode(content["content"]).decode("utf-8").strip()
+                    # Skip empty files
+                    if not decoded_content:
+                        return None
+                    return decoded_content
+                except Exception:
+                    # If decoding fails, it's probably a binary file
+                    return None
+            else:
+                # Skip binary files by returning None
+                return None
+        else:
+            file_content = content['content'].strip()
+            # Skip empty files
+            if not file_content:
+                return None
+            return file_content
+
+    def _make_request(self, url: str, max_retries: int = 3) -> requests.Response:
+        """Make a request with retry logic for rate limiting"""
+        for attempt in range(max_retries):
+            response = requests.get(url, headers=self.headers, timeout=100)
+
+            if response.status_code == 200:
+                return response
+            elif response.status_code == 403:
+                # Check if it's a rate limit issue
+                try:
+                    error_data = response.json()
+                    error_msg = error_data.get("message", "")
+
+                    # Check rate limit headers
+                    remaining = response.headers.get("X-RateLimit-Remaining", "unknown")
+                    reset_time = response.headers.get("X-RateLimit-Reset", "unknown")
+
+                    logger.warning("GitHub API 403 Error: %s", error_msg)
+                    logger.warning(
+                        "Rate limit remaining: %s, Reset time: %s", remaining, reset_time
+                    )
+
+                    if "rate limit" in error_msg.lower():
+                        if attempt < max_retries - 1:
+                            wait_time = 2 ** attempt  # Exponential backoff
+                            logger.warning(
+                                "Rate limit hit, waiting %s seconds before retry...",
+                                wait_time,
+                            )
+                            time.sleep(wait_time)
+                            continue
+
+                    # Provide helpful error message
+                    if remaining == "0":
+                        raise Exception(f"GitHub API rate limit exceeded. Please set GITHUB_ACCESS_TOKEN environment variable. Reset time: {reset_time}")
+                    else:
+                        raise Exception(f"GitHub API error: {error_msg}. This may require authentication - set GITHUB_ACCESS_TOKEN environment variable.")
+                except Exception as e:
+                    if isinstance(e, Exception) and "GitHub API" in str(e):
+                        raise
+                    # If we can't parse the response, raise the original error
+                    response.raise_for_status()
+            elif response.status_code == 401 and self.access_token:
+                # An expired or revoked PAT makes even public repos 401, which
+                # is strictly worse than not sending one. Retry unauthenticated
+                # so a stale credential degrades instead of failing the ingest.
+                logger.warning(
+                    "GitHub rejected the configured token (401); "
+                    "retrying %s unauthenticated", url,
+                )
+                anon = {"Accept": "application/vnd.github.v3+json"}
+                anon_response = requests.get(url, headers=anon, timeout=100)
+                if anon_response.status_code == 200:
+                    return anon_response
+                anon_response.raise_for_status()
+                return anon_response
+            else:
+                response.raise_for_status()
+
+        return response
+
+    def fetch_repo_files(self, repo_url: str, path: str = "") -> List[str]:
+        """Walk ``/contents/`` recursively (fallback for truncated trees)."""
+        url = f"https://api.github.com/repos/{repo_url}/contents/{path}"
+        response = self._make_request(url)
+
+        contents = response.json()
+
+        # Handle error responses from GitHub API
+        if isinstance(contents, dict) and "message" in contents:
+            raise Exception(f"GitHub API error: {contents.get('message')}")
+
+        # Ensure contents is a list
+        if not isinstance(contents, list):
+            raise TypeError(f"Expected list from GitHub API, got {type(contents).__name__}: {contents}")
+
+        files = []
+        for item in contents:
+            if item["type"] == "file":
+                files.append(item["path"])
+            elif item["type"] == "dir":
+                files.extend(self.fetch_repo_files(repo_url, item["path"]))
+        return files
+
+    def _list_candidate_files(self, repo_name: str, branch: str) -> List[str]:
+        """Enumerate ingestable paths, preferring the single tree request."""
+        try:
+            entries, truncated = self.fetch_repo_tree(repo_name, branch)
+            if not truncated:
+                return self.select_files(entries)
+            logger.warning(
+                "Tree for %s is truncated; falling back to the directory walk",
+                repo_name,
+            )
+        except Exception as e:
+            logger.warning(
+                "Tree listing failed for %s (%s); falling back to the "
+                "directory walk", repo_name, e,
+            )
+        paths = self.fetch_repo_files(repo_name)
+        return self.select_files([(p, 0) for p in paths])
+
+    def load_data(self, repo_url: str) -> List[Document]:
+        """Load every ingestable text file in ``repo_url`` as a Document."""
+        repo_name = self.normalize_repo(repo_url)
+        if not repo_name or "/" not in repo_name:
+            raise ValueError(
+                f"Not a valid GitHub repository: {repo_url!r}. "
+                "Expected a github.com URL like https://github.com/owner/name."
+            )
+        branch = self.get_default_branch(repo_name)
+        files = self._list_candidate_files(repo_name, branch)
+        logger.info(
+            "Fetching %d file(s) from %s@%s", len(files), repo_name, branch
+        )
+
+        # Fetch in parallel: this phase is pure network latency, and it was
+        # previously one blocking round-trip per file.
+        contents: Dict[str, Optional[str]] = {}
+
+        def _fetch(file_path: str) -> Tuple[str, Optional[str]]:
+            try:
+                return file_path, self.fetch_file_content(repo_name, file_path)
+            except Exception as e:
+                # One unreadable file must not sink the whole repo ingest.
+                logger.warning("Skipping %s: %s", file_path, e)
+                return file_path, None
+
+        max_workers = min(self._max_workers(), len(files)) or 1
+        with ThreadPoolExecutor(max_workers=max_workers) as pool:
+            for file_path, content in pool.map(_fetch, files):
+                contents[file_path] = content
+
+        documents = []
+        for file_path in files:
+            content = contents.get(file_path)
+            # Skip binary files (content is None)
+            if not content:
+                continue
+            documents.append(Document(
+                text=content,
+                doc_id=file_path,
+                extra_info={
+                    "title": file_path,
+                    "source": (
+                        f"https://github.com/{repo_name}/blob/{branch}/{file_path}"
+                    ),
+                }
+            ))
+        return documents

docsgpt/parser/remote/reddit_loader.py

@@ -0,0 +1,89 @@
+import json
+from typing import Any, Iterable, List
+
+from docsgpt.parser.remote.base import BaseRemote
+from docsgpt.parser.schema.base import Document
+
+REQUIRED_FIELDS = ["client_id", "client_secret", "user_agent", "search_queries"]
+
+
+class RedditPostsLoaderRemote(BaseRemote):
+    """Load Reddit posts through praw.
+
+    Create an app at https://www.reddit.com/prefs/apps/ to obtain the
+    ``client_id`` and ``client_secret``.
+    """
+
+    def load_data(self, inputs) -> List[Document]:
+        """Load posts for every configured subreddit or user.
+
+        Args:
+            inputs: JSON string holding the praw credentials plus
+                ``search_queries``, and optionally ``mode``
+                (``subreddit``/``username``), ``categories`` and
+                ``number_posts``.
+
+        Returns:
+            One document per post, carrying the post body as text.
+        """
+        try:
+            data = json.loads(inputs)
+        except json.JSONDecodeError as e:
+            raise ValueError(f"Invalid JSON input: {e}")
+
+        missing_fields = [field for field in REQUIRED_FIELDS if field not in data]
+        if missing_fields:
+            raise ValueError(f"Missing required fields: {', '.join(missing_fields)}")
+
+        mode = data.get("mode", "subreddit")
+        if mode not in ("subreddit", "username"):
+            raise ValueError(
+                "mode not correct, please enter 'username' or 'subreddit' as mode"
+            )
+        categories = data.get("categories", ["new", "hot"])
+        search_queries = data.get("search_queries")
+        number_posts = data.get("number_posts", 10)
+
+        import praw
+
+        reddit = praw.Reddit(
+            client_id=data.get("client_id"),
+            client_secret=data.get("client_secret"),
+            user_agent=data.get("user_agent"),
+        )
+
+        documents: List[Document] = []
+        for search_query in search_queries:
+            for category in categories:
+                if mode == "subreddit":
+                    listing = reddit.subreddit(search_query)
+                else:
+                    listing = reddit.redditor(search_query).submissions
+                documents.extend(
+                    self._posts_to_documents(
+                        getattr(listing, category)(limit=number_posts), category
+                    )
+                )
+        return documents
+
+    @staticmethod
+    def _posts_to_documents(posts: Iterable[Any], category: str) -> List[Document]:
+        """Convert praw submissions into documents."""
+        documents = []
+        for post in posts:
+            documents.append(
+                Document(
+                    text=post.selftext,
+                    doc_id=post.id,
+                    extra_info={
+                        "post_subreddit": post.subreddit_name_prefixed,
+                        "post_category": category,
+                        "title": post.title,
+                        "post_score": post.score,
+                        "post_id": post.id,
+                        "source": post.url,
+                        "post_author": str(post.author),
+                    },
+                )
+            )
+        return documents

docsgpt/parser/remote/remote_creator.py

@@ -0,0 +1,92 @@
+import json
+
+from docsgpt.parser.remote.sitemap_loader import SitemapLoader
+from docsgpt.parser.remote.crawler_loader import CrawlerLoader
+from docsgpt.parser.remote.web_loader import WebLoader
+from docsgpt.parser.remote.reddit_loader import RedditPostsLoaderRemote
+from docsgpt.parser.remote.github_loader import GitHubLoader
+from docsgpt.parser.remote.s3_loader import S3Loader
+
+
+class RemoteCreator:
+    """
+    Factory class for creating remote content loaders.
+
+    These loaders fetch content from remote web sources like URLs,
+    sitemaps, web crawlers, social media platforms, etc.
+
+    For external knowledge base connectors (like Google Drive),
+    use ConnectorCreator instead.
+    """
+
+    loaders = {
+        "url": WebLoader,
+        "sitemap": SitemapLoader,
+        "crawler": CrawlerLoader,
+        "reddit": RedditPostsLoaderRemote,
+        "github": GitHubLoader,
+        "s3": S3Loader,
+    }
+
+    @classmethod
+    def create_loader(cls, type, *args, **kwargs):
+        loader_class = cls.loaders.get(type.lower())
+        if not loader_class:
+            raise ValueError(f"No loader class found for type {type}")
+        return loader_class(*args, **kwargs)
+
+
+# Loader types whose load_data expects a URL string, not a config dict.
+_URL_LOADER_TYPES = {"url", "crawler", "sitemap", "github"}
+
+# Keys a remote_data dict may hold the URL under (``raw`` is the legacy shape).
+_URL_DATA_KEYS = ("url", "urls", "repo_url", "raw")
+
+
+def normalize_remote_data(source_type, remote_data):
+    """Convert a stored ``sources.remote_data`` JSONB value into the
+    ``source_data`` shape the matching loader expects.
+
+    Args:
+        source_type: The ``sources.type`` value (the loader name).
+        remote_data: The stored ``remote_data`` (dict, list, str, or None).
+
+    Returns:
+        Loader input: a URL string or list for url/crawler/sitemap/github,
+        a JSON string for reddit, a dict for s3; ``None`` when the row has
+        nothing syncable.
+    """
+    if remote_data is None:
+        return None
+
+    # Some legacy rows stored the JSON itself as a string.
+    if isinstance(remote_data, str):
+        stripped = remote_data.strip()
+        if stripped[:1] in ("{", "["):
+            try:
+                remote_data = json.loads(stripped)
+            except json.JSONDecodeError:
+                # Not actually JSON — leave remote_data as the original
+                # string; the per-loader branches below handle a string.
+                pass
+
+    loader = (source_type or "").lower()
+
+    if loader in _URL_LOADER_TYPES:
+        if isinstance(remote_data, dict):
+            for key in _URL_DATA_KEYS:
+                value = remote_data.get(key)
+                if value:
+                    return value
+            # No URL key — None keeps the loader off the dict-crash path.
+            return None
+        return remote_data
+
+    if loader == "reddit":
+        # reddit's loader runs json.loads() on its input — needs a string.
+        if isinstance(remote_data, (dict, list)):
+            return json.dumps(remote_data)
+        return remote_data
+
+    # s3's loader accepts a dict or JSON string; pass it through unchanged.
+    return remote_data

docsgpt/parser/remote/s3_loader.py

@@ -0,0 +1,440 @@
+import json
+import logging
+import os
+import tempfile
+import mimetypes
+from typing import List, Optional
+from docsgpt.core.url_validation import SSRFError, validate_url
+from docsgpt.parser.file.constants import SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS
+from docsgpt.parser.remote.base import BaseRemote
+from docsgpt.parser.schema.base import Document
+
+try:
+    import boto3
+    from botocore.exceptions import ClientError, NoCredentialsError
+except ImportError:
+    boto3 = None
+
+logger = logging.getLogger(__name__)
+
+
+# Plain-text suffixes the loader reads directly (no document parser).
+_TEXT_EXTENSIONS = {
+        ".txt",
+        ".md",
+        ".markdown",
+        ".rst",
+        ".json",
+        ".xml",
+        ".yaml",
+        ".yml",
+        ".py",
+        ".js",
+        ".ts",
+        ".jsx",
+        ".tsx",
+        ".java",
+        ".c",
+        ".cpp",
+        ".h",
+        ".hpp",
+        ".cs",
+        ".go",
+        ".rs",
+        ".rb",
+        ".php",
+        ".swift",
+        ".kt",
+        ".scala",
+        ".html",
+        ".css",
+        ".scss",
+        ".sass",
+        ".less",
+        ".sh",
+        ".bash",
+        ".zsh",
+        ".fish",
+        ".sql",
+        ".r",
+        ".m",
+        ".mat",
+        ".ini",
+        ".cfg",
+        ".conf",
+        ".config",
+        ".env",
+        ".gitignore",
+        ".dockerignore",
+        ".editorconfig",
+        ".log",
+        ".csv",
+        ".tsv",
+    }
+
+
+class S3Loader(BaseRemote):
+    """Load documents from an AWS S3 bucket."""
+
+    def __init__(self):
+        if boto3 is None:
+            raise ImportError(
+                "boto3 is required for S3Loader. Install it with: pip install boto3"
+            )
+        self.s3_client = None
+
+    def _normalize_endpoint_url(self, endpoint_url: str, bucket: str) -> tuple[str, str]:
+        """
+        Normalize endpoint URL for S3-compatible services.
+
+        Detects common mistakes like using bucket-prefixed URLs and extracts
+        the correct endpoint and bucket name.
+
+        Args:
+            endpoint_url: The provided endpoint URL
+            bucket: The provided bucket name
+
+        Returns:
+            Tuple of (normalized_endpoint_url, bucket_name)
+        """
+        import re
+        from urllib.parse import urlparse
+
+        if not endpoint_url:
+            return endpoint_url, bucket
+
+        parsed = urlparse(endpoint_url)
+        host = parsed.netloc or parsed.path
+
+        # Check for DigitalOcean Spaces bucket-prefixed URL pattern
+        # e.g., https://mybucket.nyc3.digitaloceanspaces.com
+        do_match = re.match(r"^([^.]+)\.([a-z0-9]+)\.digitaloceanspaces\.com$", host)
+        if do_match:
+            extracted_bucket = do_match.group(1)
+            region = do_match.group(2)
+            correct_endpoint = f"https://{region}.digitaloceanspaces.com"
+            logger.warning(
+                f"Detected bucket-prefixed DigitalOcean Spaces URL. "
+                f"Extracted bucket '{extracted_bucket}' from endpoint. "
+                f"Using endpoint: {correct_endpoint}"
+            )
+            # If bucket wasn't provided or differs, use extracted one
+            if not bucket or bucket != extracted_bucket:
+                logger.info(f"Using extracted bucket name: '{extracted_bucket}' (was: '{bucket}')")
+                bucket = extracted_bucket
+            return correct_endpoint, bucket
+
+        # Check for just "digitaloceanspaces.com" without region
+        if host == "digitaloceanspaces.com":
+            logger.error(
+                "Invalid DigitalOcean Spaces endpoint: missing region. "
+                "Use format: https://<region>.digitaloceanspaces.com (e.g., https://lon1.digitaloceanspaces.com)"
+            )
+
+        return endpoint_url, bucket
+
+    def _init_client(
+        self,
+        aws_access_key_id: str,
+        aws_secret_access_key: str,
+        region_name: str = "us-east-1",
+        endpoint_url: Optional[str] = None,
+        bucket: Optional[str] = None,
+    ) -> Optional[str]:
+        """
+        Initialize the S3 client with credentials.
+
+        Returns:
+            The potentially corrected bucket name if endpoint URL was normalized
+        """
+        from botocore.config import Config
+
+        client_kwargs = {
+            "aws_access_key_id": aws_access_key_id,
+            "aws_secret_access_key": aws_secret_access_key,
+            "region_name": region_name,
+        }
+
+        logger.info(f"Initializing S3 client with region: {region_name}")
+
+        corrected_bucket = bucket
+        if endpoint_url:
+            # Normalize the endpoint URL and potentially extract bucket name
+            normalized_endpoint, corrected_bucket = self._normalize_endpoint_url(endpoint_url, bucket)
+            logger.info(f"Original endpoint URL: {endpoint_url}")
+            logger.info(f"Normalized endpoint URL: {normalized_endpoint}")
+            logger.info(f"Bucket name: '{corrected_bucket}'")
+
+            try:
+                normalized_endpoint = validate_url(normalized_endpoint)
+            except SSRFError as e:
+                raise ValueError(f"Invalid S3 endpoint_url: {e}") from e
+
+            client_kwargs["endpoint_url"] = normalized_endpoint
+            # Use path-style addressing for S3-compatible services
+            # (DigitalOcean Spaces, MinIO, etc.)
+            client_kwargs["config"] = Config(s3={"addressing_style": "path"})
+        else:
+            logger.info("Using default AWS S3 endpoint")
+
+        self.s3_client = boto3.client("s3", **client_kwargs)
+        logger.info("S3 client initialized successfully")
+
+        return corrected_bucket
+
+    def is_text_file(self, file_path: str) -> bool:
+        """Determine if a file is a text file based on extension."""
+        text_extensions = _TEXT_EXTENSIONS
+
+        file_lower = file_path.lower()
+        for ext in text_extensions:
+            if file_lower.endswith(ext):
+                return True
+
+        mime_type, _ = mimetypes.guess_type(file_path)
+        if mime_type and (
+            mime_type.startswith("text")
+            or mime_type in ["application/json", "application/xml"]
+        ):
+            return True
+
+        return False
+
+    def is_supported_document(self, file_path: str) -> bool:
+        """Check if file is a supported document type for parsing."""
+        # The upload whitelist plus the historical S3 list, so an object the
+        # upload API accepts is never silently skipped here.
+        document_extensions = (set(SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS) - _TEXT_EXTENSIONS) | {
+            ".pdf",
+            ".docx",
+            ".doc",
+            ".xlsx",
+            ".xls",
+            ".pptx",
+            ".ppt",
+            ".epub",
+            ".odt",
+            ".rtf",
+        }
+
+        file_lower = file_path.lower()
+        for ext in document_extensions:
+            if file_lower.endswith(ext):
+                return True
+
+        return False
+
+    def list_objects(self, bucket: str, prefix: str = "") -> List[str]:
+        """
+        List all objects in the bucket with the given prefix.
+
+        Args:
+            bucket: S3 bucket name
+            prefix: Optional path prefix to filter objects
+
+        Returns:
+            List of object keys
+        """
+        objects = []
+        paginator = self.s3_client.get_paginator("list_objects_v2")
+
+        logger.info(f"Listing objects in bucket: '{bucket}' with prefix: '{prefix}'")
+        logger.debug(f"S3 client endpoint: {self.s3_client.meta.endpoint_url}")
+
+        try:
+            page_count = 0
+            for page in paginator.paginate(Bucket=bucket, Prefix=prefix):
+                page_count += 1
+                logger.debug(f"Processing page {page_count}, keys in response: {list(page.keys())}")
+                if "Contents" in page:
+                    for obj in page["Contents"]:
+                        key = obj["Key"]
+                        if not key.endswith("/"):
+                            objects.append(key)
+                            logger.debug(f"Found object: {key}")
+                else:
+                    logger.info(f"Page {page_count} has no 'Contents' key - bucket may be empty or prefix not found")
+
+            logger.info(f"Found {len(objects)} objects in bucket '{bucket}'")
+
+        except ClientError as e:
+            error_code = e.response.get("Error", {}).get("Code", "")
+            error_message = e.response.get("Error", {}).get("Message", "")
+            logger.error(f"ClientError listing objects - Code: {error_code}, Message: {error_message}")
+            logger.error(f"Full error response: {e.response}")
+            logger.error(f"Bucket: '{bucket}', Prefix: '{prefix}', Endpoint: {self.s3_client.meta.endpoint_url}")
+
+            if error_code == "NoSuchBucket":
+                raise Exception(f"S3 bucket '{bucket}' does not exist")
+            elif error_code == "AccessDenied":
+                raise Exception(
+                    f"Access denied to S3 bucket '{bucket}'. Check your credentials and permissions."
+                )
+            elif error_code == "NoSuchKey":
+                # This is unusual for ListObjectsV2 - may indicate endpoint/bucket configuration issue
+                logger.error(
+                    "NoSuchKey error on ListObjectsV2 - this may indicate the bucket name "
+                    "is incorrect or the endpoint URL format is wrong. "
+                    "For DigitalOcean Spaces, the endpoint should be like: "
+                    "https://<region>.digitaloceanspaces.com and bucket should be just the space name."
+                )
+                raise Exception(
+                    f"S3 error: {e}. For S3-compatible services, verify: "
+                    f"1) Endpoint URL format (e.g., https://nyc3.digitaloceanspaces.com), "
+                    f"2) Bucket name is just the space/bucket name without region prefix"
+                )
+            else:
+                raise Exception(f"S3 error: {e}")
+        except NoCredentialsError:
+            raise Exception(
+                "AWS credentials not found. Please provide valid credentials."
+            )
+
+        return objects
+
+    def get_object_content(self, bucket: str, key: str) -> Optional[str]:
+        """
+        Get the content of an S3 object as text.
+
+        Args:
+            bucket: S3 bucket name
+            key: Object key
+
+        Returns:
+            File content as string, or None if file should be skipped
+        """
+        if not self.is_text_file(key) and not self.is_supported_document(key):
+            return None
+
+        try:
+            response = self.s3_client.get_object(Bucket=bucket, Key=key)
+            content = response["Body"].read()
+
+            if self.is_text_file(key):
+                try:
+                    decoded_content = content.decode("utf-8").strip()
+                    if not decoded_content:
+                        return None
+                    return decoded_content
+                except UnicodeDecodeError:
+                    return None
+            elif self.is_supported_document(key):
+                return self._process_document(content, key)
+
+        except ClientError as e:
+            error_code = e.response.get("Error", {}).get("Code", "")
+            if error_code == "NoSuchKey":
+                return None
+            elif error_code == "AccessDenied":
+                print(f"Access denied to object: {key}")
+                return None
+            else:
+                print(f"Error fetching object {key}: {e}")
+                return None
+
+        return None
+
+    def _process_document(self, content: bytes, key: str) -> Optional[str]:
+        """
+        Process a document file (PDF, DOCX, etc.) and extract text.
+
+        Args:
+            content: File content as bytes
+            key: Object key (filename)
+
+        Returns:
+            Extracted text content
+        """
+        ext = os.path.splitext(key)[1].lower()
+
+        with tempfile.NamedTemporaryFile(suffix=ext, delete=False) as tmp_file:
+            tmp_file.write(content)
+            tmp_path = tmp_file.name
+
+        try:
+            from docsgpt.parser.file.bulk import SimpleDirectoryReader
+
+            reader = SimpleDirectoryReader(input_files=[tmp_path])
+            documents = reader.load_data()
+            if documents:
+                return "\n\n".join(doc.text for doc in documents if doc.text)
+            return None
+        except Exception as e:
+            print(f"Error processing document {key}: {e}")
+            return None
+        finally:
+            if os.path.exists(tmp_path):
+                os.unlink(tmp_path)
+
+    def load_data(self, inputs) -> List[Document]:
+        """
+        Load documents from an S3 bucket.
+
+        Args:
+            inputs: JSON string or dict containing:
+                - aws_access_key_id: AWS access key ID
+                - aws_secret_access_key: AWS secret access key
+                - bucket: S3 bucket name
+                - prefix: Optional path prefix to filter objects
+                - region: AWS region (default: us-east-1)
+                - endpoint_url: Custom S3 endpoint URL (for MinIO, R2, etc.)
+
+        Returns:
+            List of Document objects
+        """
+        if isinstance(inputs, str):
+            try:
+                data = json.loads(inputs)
+            except json.JSONDecodeError as e:
+                raise ValueError(f"Invalid JSON input: {e}")
+        else:
+            data = inputs
+
+        required_fields = ["aws_access_key_id", "aws_secret_access_key", "bucket"]
+        missing_fields = [field for field in required_fields if not data.get(field)]
+        if missing_fields:
+            raise ValueError(f"Missing required fields: {', '.join(missing_fields)}")
+
+        aws_access_key_id = data["aws_access_key_id"]
+        aws_secret_access_key = data["aws_secret_access_key"]
+        bucket = data["bucket"]
+        prefix = data.get("prefix", "")
+        region = data.get("region", "us-east-1")
+        endpoint_url = data.get("endpoint_url", "")
+
+        logger.info(f"Loading data from S3 - Bucket: '{bucket}', Prefix: '{prefix}', Region: '{region}'")
+        if endpoint_url:
+            logger.info(f"Custom endpoint URL provided: '{endpoint_url}'")
+
+        corrected_bucket = self._init_client(
+            aws_access_key_id, aws_secret_access_key, region, endpoint_url or None, bucket
+        )
+
+        # Use the corrected bucket name if endpoint URL normalization extracted one
+        if corrected_bucket and corrected_bucket != bucket:
+            logger.info(f"Using corrected bucket name: '{corrected_bucket}' (original: '{bucket}')")
+            bucket = corrected_bucket
+
+        objects = self.list_objects(bucket, prefix)
+        documents = []
+
+        for key in objects:
+            content = self.get_object_content(bucket, key)
+            if content is None:
+                continue
+
+            documents.append(
+                Document(
+                    text=content,
+                    doc_id=key,
+                    extra_info={
+                        "title": os.path.basename(key),
+                        "source": f"s3://{bucket}/{key}",
+                        "bucket": bucket,
+                        "key": key,
+                    },
+                )
+            )
+
+        logger.info(f"Loaded {len(documents)} documents from S3 bucket '{bucket}'")
+        return documents

docsgpt/parser/remote/sitemap_loader.py

@@ -0,0 +1,111 @@
+import logging
+import re
+
+import defusedxml.ElementTree as ET
+from bs4 import BeautifulSoup
+
+from docsgpt.parser.remote.base import BaseRemote
+from docsgpt.parser.schema.base import Document
+from docsgpt.core.url_validation import validate_url, SSRFError
+from docsgpt.security.safe_url import UnsafeUserUrlError, pinned_request
+
+class SitemapLoader(BaseRemote):
+    def __init__(self, limit=20):
+        self.limit = limit  # Adding limit to control the number of URLs to process
+
+    def load_data(self, inputs):
+        sitemap_url= inputs
+        # Check if the input is a list and if it is, use the first element
+        if isinstance(sitemap_url, list) and sitemap_url:
+            sitemap_url = sitemap_url[0]
+
+        # Validate URL to prevent SSRF attacks
+        try:
+            sitemap_url = validate_url(sitemap_url)
+        except SSRFError as e:
+            logging.error(f"URL validation failed: {e}")
+            return []
+
+        urls = self._extract_urls(sitemap_url)
+        if not urls:
+            print(f"No URLs found in the sitemap: {sitemap_url}")
+            return []
+
+        # Load content of extracted URLs
+        documents = []
+        processed_urls = 0  # Counter for processed URLs
+        for url in urls:
+            if self.limit is not None and processed_urls >= self.limit:
+                break  # Stop processing if the limit is reached
+
+            try:
+                url = validate_url(url)
+            except SSRFError as e:
+                logging.error(f"URL validation failed for sitemap entry {url}: {e}")
+                continue
+            try:
+                response = pinned_request("GET", url, timeout=30)
+                response.raise_for_status()
+                soup = BeautifulSoup(response.text, "html.parser")
+                documents.append(
+                    Document(
+                        soup.get_text(separator="\n", strip=True),
+                        extra_info={"source": url},
+                    )
+                )
+                processed_urls += 1  # Increment the counter after processing each URL
+            except Exception as e:
+                logging.error(f"Error processing URL {url}: {e}", exc_info=True)
+                continue
+
+        return documents
+
+    def _extract_urls(self, sitemap_url):
+        try:
+            response = pinned_request("GET", sitemap_url, timeout=30)
+            response.raise_for_status()
+        except UnsafeUserUrlError as e:
+            print(f"URL validation failed for sitemap: {sitemap_url}. Error: {e}")
+            return []
+        except Exception as e:
+            print(f"Failed to fetch sitemap: {sitemap_url}. Error: {e}")
+            return []
+
+        # Determine if this is a sitemap or a URL
+        if self._is_sitemap(response):
+            # It's a sitemap, so parse it and extract URLs
+            return self._parse_sitemap(response.content)
+        else:
+            # It's not a sitemap, return the URL itself
+            return [sitemap_url]
+
+    def _is_sitemap(self, response):
+        content_type = response.headers.get('Content-Type', '')
+        if 'xml' in content_type or response.url.endswith('.xml'):
+            return True
+
+        if '<sitemapindex' in response.text or '<urlset' in response.text:
+            return True
+
+        return False
+
+    def _parse_sitemap(self, sitemap_content):
+        # Remove namespaces
+        sitemap_content = re.sub(' xmlns="[^"]+"', '', sitemap_content.decode('utf-8'), count=1)
+
+        root = ET.fromstring(sitemap_content)
+
+        urls = []
+        for loc in root.findall('.//url/loc'):
+            if not loc.text:
+                continue
+            urls.append(loc.text)
+
+        # Check for nested sitemaps
+        for sitemap in root.findall('.//sitemap/loc'):
+            nested_sitemap_url = sitemap.text
+            if not nested_sitemap_url:
+                continue
+            urls.extend(self._extract_urls(nested_sitemap_url))
+
+        return urls

docsgpt/parser/remote/web_loader.py

@@ -0,0 +1,57 @@
+import logging
+
+from bs4 import BeautifulSoup
+
+from docsgpt.core.url_validation import SSRFError, validate_url
+from docsgpt.parser.remote.base import BaseRemote
+from docsgpt.parser.schema.base import Document
+from docsgpt.security.safe_url import pinned_request
+
+headers = {
+    "User-Agent": "Mozilla/5.0",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*"
+    ";q=0.8",
+    "Accept-Language": "en-US,en;q=0.5",
+    "Referer": "https://www.google.com/",
+    "DNT": "1",
+    "Connection": "keep-alive",
+    "Upgrade-Insecure-Requests": "1",
+}
+
+
+class WebLoader(BaseRemote):
+    def load_data(self, inputs):
+        urls = inputs
+        if isinstance(urls, str):
+            urls = [urls]
+        documents = []
+        for url in urls:
+            try:
+                url = validate_url(url)
+            except SSRFError as e:
+                logging.warning(
+                    f"Skipping URL due to SSRF validation failure: {url} - {e}"
+                )
+                continue
+            try:
+                response = pinned_request("GET", url, headers=headers, timeout=30)
+                response.raise_for_status()
+                soup = BeautifulSoup(response.text, "html.parser")
+                metadata = {"source": url}
+                if soup.title:
+                    title = soup.title.get_text(strip=True)
+                    if title:
+                        metadata["title"] = title
+                html_tag = soup.find("html")
+                if html_tag and html_tag.get("lang"):
+                    metadata["language"] = html_tag.get("lang")
+                documents.append(
+                    Document(
+                        soup.get_text(separator="\n", strip=True),
+                        extra_info=metadata,
+                    )
+                )
+            except Exception as e:
+                logging.error(f"Error processing URL {url}: {e}", exc_info=True)
+                continue
+        return documents

docsgpt/parser/schema/__init__.py

@@ -0,0 +1 @@
+

docsgpt/parser/schema/base.py

@@ -0,0 +1,34 @@
+"""Base schema for readers."""
+from dataclasses import dataclass
+
+from docsgpt.parser.schema.schema import BaseDocument
+from docsgpt.vectorstore.document_class import Document as VectorDocument
+
+
+@dataclass
+class Document(BaseDocument):
+    """Generic interface for a data document.
+
+    This document connects to data sources.
+
+    """
+
+    def __post_init__(self) -> None:
+        """Post init."""
+        if self.text is None:
+            raise ValueError("text field not set.")
+
+    @classmethod
+    def get_type(cls) -> str:
+        """Get Document type."""
+        return "Document"
+
+    def to_vector_format(self) -> VectorDocument:
+        """Convert struct to the page_content/metadata shape vector stores take."""
+        metadata = self.extra_info or {}
+        return VectorDocument(page_content=self.text, metadata=metadata)
+
+    @classmethod
+    def from_vector_format(cls, doc: VectorDocument) -> "Document":
+        """Convert struct from the vector-store document shape."""
+        return cls(text=doc.page_content, extra_info=doc.metadata)

docsgpt/parser/schema/schema.py

@@ -0,0 +1,64 @@
+"""Base schema for data structures."""
+from abc import abstractmethod
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
+
+from dataclasses_json import DataClassJsonMixin
+
+
+@dataclass
+class BaseDocument(DataClassJsonMixin):
+    """Base document.
+
+    Generic abstract interfaces that captures both index structs
+    as well as documents.
+
+    """
+
+    # TODO: consolidate fields from Document/IndexStruct into base class
+    text: Optional[str] = None
+    doc_id: Optional[str] = None
+    embedding: Optional[List[float]] = None
+
+    # extra fields
+    extra_info: Optional[Dict[str, Any]] = None
+
+    @classmethod
+    @abstractmethod
+    def get_type(cls) -> str:
+        """Get Document type."""
+
+    def get_text(self) -> str:
+        """Get text."""
+        if self.text is None:
+            raise ValueError("text field not set.")
+        return self.text
+
+    def get_doc_id(self) -> str:
+        """Get doc_id."""
+        if self.doc_id is None:
+            raise ValueError("doc_id not set.")
+        return self.doc_id
+
+    @property
+    def is_doc_id_none(self) -> bool:
+        """Check if doc_id is None."""
+        return self.doc_id is None
+
+    def get_embedding(self) -> List[float]:
+        """Get embedding.
+
+        Errors if embedding is None.
+
+        """
+        if self.embedding is None:
+            raise ValueError("embedding not set.")
+        return self.embedding
+
+    @property
+    def extra_info_str(self) -> Optional[str]:
+        """Extra info string."""
+        if self.extra_info is None:
+            return None
+
+        return "\n".join([f"{k}: {str(v)}" for k, v in self.extra_info.items()])

docsgpt/parser/tokenization.py

@@ -0,0 +1,291 @@
+"""Counting and splitting text in the embedding model's own tokenizer.
+
+Chunk sizes only mean something in the units the embedding server counts. The
+chunker used to count cl100k (tiktoken) while the server counted whatever its
+model used, so ``max_tokens`` was a value in one unit compared against a limit
+in another. Every recalibration of that number was really an attempt to guess
+the conversion factor.
+
+This module removes the conversion. :func:`get_token_counter` returns a counter
+backed by the configured embedding model's tokenizer, falling back to cl100k
+when that tokenizer cannot be loaded -- an offline install, or a model the
+registry does not describe.
+
+Splitting never round-trips through ``decode``. Byte-level BPE decodes
+losslessly, but WordPiece tokenizers do not: all-mpnet-base-v2 lowercases, so
+decoding ``"Hello World"`` yields ``"hello world"`` and would silently rewrite
+every stored document. Counters therefore cut the *original* string at
+character offsets the tokenizer reports.
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+from typing import Iterator, List, Optional, Tuple
+
+from docsgpt.core.settings import settings
+from docsgpt.utils import get_encoding
+from docsgpt.vectorstore.model_registry import resolve
+
+logger = logging.getLogger(__name__)
+
+_cache: dict = {}
+_cache_lock = threading.Lock()
+
+
+def _windows(total: int, first: int, rest: int) -> Iterator[Tuple[int, int]]:
+    """Yield ``(start, end)`` token windows, the first sized independently.
+
+    A header consumes part of the first chunk's budget but none of the rest,
+    so the first window is often smaller than those that follow.
+    """
+    start = 0
+    budget = max(1, first)
+    while start < total:
+        end = min(start + budget, total)
+        yield start, end
+        start = end
+        budget = max(1, rest)
+
+
+# A token spanning more characters than this collapsed a run the tokenizer
+# could not break up: WordPiece emits a single ``[UNK]`` for any word longer
+# than ``max_input_chars_per_word``. Charging that once makes a base64 blob or
+# a minified bundle look tiny, so nothing splits it and an oversized chunk
+# reaches the embedding server. Real tokens are a few characters, so prose
+# never reaches this bound.
+_MAX_CHARS_PER_TOKEN = 16
+
+
+def _token_weight(start: int, end: int) -> int:
+    """Tokens a span costs, charging a collapsed run by its length."""
+    span = end - start
+    return max(1, -(-span // _MAX_CHARS_PER_TOKEN))
+
+
+def _cap_piece_chars(pieces: List[str], first: int, rest: int) -> List[str]:
+    """Cut any piece holding more characters than its budget can cover."""
+    capped: List[str] = []
+    budget = first
+    for piece in pieces:
+        limit = max(1, budget * _MAX_CHARS_PER_TOKEN)
+        while len(piece) > limit:
+            capped.append(piece[:limit])
+            piece = piece[limit:]
+            budget = rest
+            limit = max(1, budget * _MAX_CHARS_PER_TOKEN)
+        capped.append(piece)
+        budget = rest
+    return capped
+
+
+class TokenCounter:
+    """Counts and splits text in one tokenizer's units.
+
+    Attributes:
+        name: Human-readable identifier of the underlying tokenizer, for logs.
+    """
+
+    name = "unknown"
+
+    def count(self, text: str) -> int:
+        """Number of tokens ``text`` occupies."""
+        raise NotImplementedError
+
+    def split(
+        self, text: str, max_tokens: int, first_max_tokens: Optional[int] = None
+    ) -> List[str]:
+        """Cut ``text`` into consecutive pieces of at most ``max_tokens``.
+
+        The concatenation of the returned pieces equals ``text`` exactly.
+
+        Args:
+            text: Source text.
+            max_tokens: Token budget per piece; values below 1 are treated as 1.
+            first_max_tokens: Budget for the first piece only, when it must
+                leave room for a header. Defaults to ``max_tokens``.
+
+        Returns:
+            The pieces, in order. An empty ``text`` yields an empty list.
+        """
+        raise NotImplementedError
+
+
+class TiktokenCounter(TokenCounter):
+    """cl100k counter -- the historical behaviour, and the fallback."""
+
+    name = "cl100k_base"
+
+    def __init__(self) -> None:
+        self._encoding = get_encoding()
+
+    def count(self, text: str) -> int:
+        if not text:
+            return 0
+        return len(self._encoding.encode_ordinary(text))
+
+    def split(
+        self, text: str, max_tokens: int, first_max_tokens: Optional[int] = None
+    ) -> List[str]:
+        if not text:
+            return []
+        rest = max(1, max_tokens)
+        first = max(1, first_max_tokens if first_max_tokens is not None else rest)
+        tokens = self._encoding.encode_ordinary(text)
+        if len(tokens) <= first:
+            return [text]
+
+        # Cut the original string at the character offsets the tokenizer
+        # reports, the way :class:`HuggingFaceCounter` does. Decoding each
+        # window on its own instead splits any multi-byte character that
+        # straddles a boundary across two byte sequences, and each half decodes
+        # to U+FFFD -- roughly one boundary in five on CJK text, silently
+        # destroying a character per cut.
+        offsets = self._encoding.decode_with_offsets(tokens)[1]
+        end_of_text = len(text)
+        pieces: List[str] = []
+        cursor = 0
+        for _, end in _windows(len(tokens), first, rest):
+            # Offsets index into the decoded string. ``encode_ordinary``
+            # round-trips for anything cl100k can represent, so it is ``text``;
+            # clamping keeps the cuts in range if it ever is not, and slicing
+            # ``text`` monotonically keeps the pieces reassembling exactly
+            # either way.
+            end_char = end_of_text if end >= len(offsets) else min(offsets[end], end_of_text)
+            if end_char <= cursor:
+                continue
+            pieces.append(text[cursor:end_char])
+            cursor = end_char
+        if cursor < end_of_text:
+            if pieces:
+                pieces[-1] = pieces[-1] + text[cursor:]
+            else:
+                pieces.append(text[cursor:])
+        return pieces
+
+
+class HuggingFaceCounter(TokenCounter):
+    """Counts in a Hugging Face tokenizer, slicing by character offsets."""
+
+    def __init__(self, tokenizer, name: str) -> None:
+        self._tokenizer = tokenizer
+        self.name = name
+
+    def _encode(self, text: str):
+        return self._tokenizer.encode(text, add_special_tokens=False)
+
+    def count(self, text: str) -> int:
+        if not text:
+            return 0
+        encoding = self._encode(text)
+        if not encoding.offsets:
+            return len(encoding.ids)
+        return sum(_token_weight(start, end) for start, end in encoding.offsets)
+
+    def split(
+        self, text: str, max_tokens: int, first_max_tokens: Optional[int] = None
+    ) -> List[str]:
+        if not text:
+            return []
+        rest = max(1, max_tokens)
+        first = max(1, first_max_tokens if first_max_tokens is not None else rest)
+        offsets = self._encode(text).offsets
+        if self.count(text) <= first:
+            return [text]
+
+        pieces: List[str] = []
+        cursor = 0
+        for start_token, end_token in _windows(len(offsets), first, rest):
+            window = offsets[start_token:end_token]
+            # Some tokenizers emit (0, 0) for specials or normalised-away
+            # characters; those carry no span to cut on.
+            spans = [end for _, end in window if end > cursor]
+            if not spans:
+                continue
+            end_char = max(spans)
+            pieces.append(text[cursor:end_char])
+            cursor = end_char
+        if cursor < len(text):
+            # Trailing characters the tokenizer dropped (e.g. whitespace) belong
+            # to the final piece, so no input is lost.
+            if pieces:
+                pieces[-1] = pieces[-1] + text[cursor:]
+            else:
+                pieces.append(text[cursor:])
+        return _cap_piece_chars(pieces, first, rest)
+
+
+def _tokenizer_file(repo: str) -> str:
+    """Path to ``repo``'s ``tokenizer.json``, from the hub cache when present.
+
+    A warmed cache (the Docker image bakes the default models) answers without
+    touching the network. ``hf_hub_download`` would otherwise revalidate the
+    revision with a HEAD request on every process start, and stall for the
+    etag timeout on a host that cannot reach huggingface.co.
+    """
+    from huggingface_hub import hf_hub_download
+
+    try:
+        return hf_hub_download(repo, "tokenizer.json", local_files_only=True)
+    except Exception:  # noqa: BLE001 -- not cached: fetch it
+        return hf_hub_download(repo, "tokenizer.json")
+
+
+def _load_hf_counter(repo: str) -> Optional[HuggingFaceCounter]:
+    """Load ``repo``'s tokenizer, or ``None`` if it is not reachable."""
+    try:
+        from tokenizers import Tokenizer
+
+        tokenizer = Tokenizer.from_file(_tokenizer_file(repo))
+        # Repos ship padding and truncation defaults meant for inference
+        # batches. Left on, every count returns the padded width (128 for
+        # mpnet) and the offsets carry (0, 0) entries for the padding, so both
+        # counting and slicing are wrong.
+        tokenizer.no_padding()
+        tokenizer.no_truncation()
+        return HuggingFaceCounter(tokenizer, repo)
+    except Exception as exc:  # noqa: BLE001 -- chunking must never hard-fail here
+        logger.warning(
+            "Could not load the tokenizer for %s (%s); counting chunk sizes in "
+            "cl100k instead. Chunk sizes will be approximate for this model.",
+            repo,
+            exc,
+        )
+        return None
+
+
+def get_token_counter(embeddings_name: Optional[str] = None) -> TokenCounter:
+    """Return the counter for ``embeddings_name``, cached per process.
+
+    Args:
+        embeddings_name: Model name; defaults to ``settings.EMBEDDINGS_NAME``.
+
+    Returns:
+        A :class:`HuggingFaceCounter` for a model whose tokenizer could be
+        loaded, else a :class:`TiktokenCounter`.
+    """
+    name = embeddings_name or getattr(settings, "EMBEDDINGS_NAME", None)
+    key = name or "__default__"
+    with _cache_lock:
+        if key in _cache:
+            return _cache[key]
+
+    spec = resolve(name)
+    repo = spec.repo if spec else name
+    counter: TokenCounter
+    if repo and (spec is None or spec.provider == "fastembed"):
+        counter = _load_hf_counter(repo) or TiktokenCounter()
+    else:
+        counter = TiktokenCounter()
+
+    with _cache_lock:
+        _cache[key] = counter
+    logger.info("Chunking will count tokens with %s", counter.name)
+    return counter
+
+
+def reset_cache() -> None:
+    """Drop cached counters. For tests and for a settings change at runtime."""
+    with _cache_lock:
+        _cache.clear()