Merge pull request #2701 from arc53/anydoc-support

AGENTS.md

@@ -29,6 +29,8 @@ Use these commands once the dev prerequisites above are satisfied.
 ```bash
 source .venv/bin/activate  # macOS/Linux
 uv pip install -r application/requirements.txt  # or: pip install -r application/requirements.txt
+# Optional docling parser engine (OCR / read_document structured output):
+# uv pip install -r application/requirements-docling.txt
 ```
 
 Run the API. For local dev, prefer the ASGI entrypoint under uvicorn — it

application/Dockerfile

@@ -23,8 +23,8 @@ RUN wget -q -O - https://sh.rustup.rs | sh -s -- -y
 # Clean up to reduce container size
 RUN apt-get remove --purge -y wget unzip && apt-get autoremove -y && rm -rf /var/lib/apt/lists/*
 
-# Copy requirements.txt
-COPY requirements.txt .
+# Copy requirements manifests
+COPY requirements.txt requirements-docling.txt ./
 
 # Setup Python virtual environment
 RUN python3.12 -m venv /venv
@@ -37,6 +37,15 @@ RUN pip install --no-cache-dir --upgrade pip && \
     pip install --no-cache-dir tiktoken && \
     pip install --no-cache-dir -r requirements.txt
 
+# Optional docling parser engine (DOC_PARSER_ENGINE=docling, the docling OCR
+# backend, read_document's structured output) — OFF by default: it pulls the
+# layout/OCR model stack and adds gigabytes to the image. anydoc (in
+# requirements.txt) is the default parser and needs none of it, and OCR runs
+# natively on tesseract (below, also opt-in) without it.
+ARG INSTALL_DOCLING=false
+RUN if [ "$INSTALL_DOCLING" = "true" ]; then \
+        pip install --no-cache-dir -r requirements-docling.txt; \
+    fi
 
 # Final Stage
 FROM ubuntu:24.04 as final
@@ -53,6 +62,20 @@ RUN apt-get update && \
     ln -s /usr/bin/python3.12 /usr/bin/python && \
     rm -rf /var/lib/apt/lists/*
 
+# Optional tesseract OCR engine (OCR_ENABLED=true with OCR_ENGINE=tesseract,
+# the default engine) — OFF by default like every other OCR dependency; OCR
+# itself is off unless configured. Opt in with --build-arg
+# INSTALL_TESSERACT=true (setup.sh writes it to .env when OCR is enabled);
+# ~35 MB of system packages. Extra language packs are a deployment concern
+# (apt: tesseract-ocr-<lang>, then list them in OCR_LANGS). A DeepSeek-OCR
+# endpoint (OCR_ENGINE=deepseek) needs none of this.
+ARG INSTALL_TESSERACT=false
+RUN if [ "$INSTALL_TESSERACT" = "true" ]; then \
+        apt-get update && \
+        apt-get install -y --no-install-recommends tesseract-ocr tesseract-ocr-eng && \
+        rm -rf /var/lib/apt/lists/*; \
+    fi
+
 # Set working directory
 WORKDIR /app
 

application/agents/tools/read_document.py

@@ -102,7 +102,8 @@ class ReadDocumentTool(Tool):
                             "enum": ["markdown", "text", "structured", "chunks"],
                             "description": "Shape of the parsed result (default: markdown). Note: "
                             "`structured` always uses the Docling engine regardless of `engine` "
-                            "(the `fast` engine is markdown/text only).",
+                            "(requires the optional docling install; the other engines are "
+                            "markdown/text only).",
                         },
                         "ocr": {
                             "type": "string",
@@ -115,9 +116,11 @@ class ReadDocumentTool(Tool):
                         },
                         "engine": {
                             "type": "string",
-                            "enum": ["auto", "docling", "fast"],
-                            "description": "Parser engine (default: auto). `fast` is a lighter "
-                            "markdown/text-only engine; it is ignored when `output='structured'`, "
+                            "enum": ["auto", "docling", "anydoc", "fast"],
+                            "description": "Parser engine (default: auto = the server's configured "
+                            "engine). `anydoc` is the fast Markdown converter, `docling` the "
+                            "layout-model engine (optional install, needed for tables/OCR), "
+                            "`fast` a plain-text legacy path. Ignored when `output='structured'`, "
                             "which always uses Docling.",
                         },
                         "max_chars": {

application/api/user/sources/upload.py

@@ -272,7 +272,21 @@ class UploadFile(Resource):
                     # Office/e-book containers are ZIP-based formats but must
                     # be parsed as documents, not expanded as user archives.
                     is_office_format = safe_file.lower().endswith(
-                        (".docx", ".xlsx", ".pptx", ".odt", ".ods", ".odp", ".epub")
+                        (
+                            ".docx",
+                            ".docm",
+                            ".xlsx",
+                            ".xlsm",
+                            ".xlsb",
+                            ".pptx",
+                            ".pptm",
+                            ".ppsx",
+                            ".ppsm",
+                            ".odt",
+                            ".ods",
+                            ".odp",
+                            ".epub",
+                        )
                     )
                     if zipfile.is_zipfile(temp_file_path) and not is_office_format:
                         extract_dir = os.path.join(upload_dir, "extracted")

application/core/settings.py

@@ -2,7 +2,7 @@ import os
 from pathlib import Path
 from typing import Optional
 
-from pydantic import Field, field_validator
+from pydantic import AliasChoices, Field, field_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@@ -118,20 +118,106 @@ class Settings(BaseSettings):
     UPLOAD_MAX_ARCHIVE_DEPTH: int = Field(default=3, ge=0)
     PARSE_PDF_AS_IMAGE: bool = False
     PARSE_IMAGE_REMOTE: bool = False
-    DOCLING_OCR_ENABLED: bool = False  # Enable OCR for docling parsers (PDF, images)
-    DOCLING_OCR_ATTACHMENTS_ENABLED: bool = False  # Enable OCR for docling when parsing attachments
-    # Pages docling buffers in flight; its default of 100 drives worker RSS to ~3 GB on a mid-size PDF.
+    # Document parser for source ingestion, chat attachments and the
+    # read_document tool. "anydoc" (default): firecrawl-anydoc, a Rust
+    # converter with no ML models — milliseconds per file, ~100 MB peak RSS.
+    # "docling": the layout/table-model pipeline (optional install; needed
+    # for read_document's structured output and the docling OCR backend).
+    # Files anydoc cannot convert (scanned PDFs, malformed input) fall back to
+    # docling when it is installed, otherwise to the native OCR parsers (OCR
+    # on) or the legacy parsers. Rollback to the previous behaviour is this
+    # one variable.
+    DOC_PARSER_ENGINE: str = "anydoc"
+    # OCR for scanned PDFs and images. OCR_ENABLED covers source ingestion,
+    # OCR_ATTACHMENTS_ENABLED chat attachments. Which stack performs it is
+    # OCR_BACKEND; which engine, OCR_ENGINE. The DOCLING_OCR_* names are the
+    # pre-2026-09 spellings and stay accepted as aliases.
+    OCR_ENABLED: bool = Field(
+        default=False, validation_alias=AliasChoices("OCR_ENABLED", "DOCLING_OCR_ENABLED")
+    )
+    OCR_ATTACHMENTS_ENABLED: bool = Field(
+        default=False,
+        validation_alias=AliasChoices("OCR_ATTACHMENTS_ENABLED", "DOCLING_OCR_ATTACHMENTS_ENABLED"),
+    )
+    # Which stack runs OCR when it is on:
+    #   auto    — docling when installed, otherwise native.
+    #   docling — the layout-model pipeline (hybrid region OCR, reading order,
+    #             table structure); needs the optional docling extra.
+    #   native  — pypdfium2/Pillow page rendering straight into tesseract or a
+    #             DeepSeek-OCR endpoint (application/parser/file/ocr_parser.py).
+    #             No ML models in the worker; tables come out as text lines
+    #             under tesseract.
+    OCR_BACKEND: str = "auto"
+    # Pages docling's threaded pipeline buffers in flight; the library
+    # default (100) drives worker RSS to ~3 GB on a mid-size PDF.
     DOCLING_PIPELINE_QUEUE_MAX_SIZE: int = 2
     DOCLING_COMPILE_TORCH_MODELS: bool = False
     DOCLING_TABULAR_MAX_BYTES: int = 2_000_000
     DOCLING_MARKUP_MAX_BYTES: int = 8_000_000
-    # Chars-per-page floor below which an OCR'd parse is treated as a docling dropout rather than
-    # content: retried once on a fresh converter, then failed loudly instead of indexing an empty
-    # document. Long-running workers were seen returning zero chars per page with no error. 0 disables.
-    DOCLING_OCR_MIN_CHARS_PER_PAGE: int = 20
-    # Read PDF *attachments* via their text layer (pypdfium2), falling back to docling when there
-    # is none. Attachments go into a prompt, where docling's structure does not earn its tens of
-    # seconds per file. Source ingestion is unaffected and always uses docling.
+    # HTML/XHTML larger than this (bytes) are head-truncated before the
+    # markdownify parser runs (the anydoc engine's HTML path). The tree that
+    # path builds costs ~50x the input — 30 MB of HTML measured at 1.6 GB RSS —
+    # and the upload cap is 100 MB, so the gate is what keeps one upload from
+    # taking the ingest worker down. 0 disables it.
+    MARKUP_MAX_BYTES: int = 8_000_000
+    # Trust-check anydoc's PDF output (application/parser/file/pdf_trust.py):
+    # flag composite (Type0) fonts without a ToUnicode map, and CJK-declaring
+    # PDFs whose extracted text has almost no CJK — the two classes where
+    # anydoc drops text silently. A flagged file re-parses on the docling
+    # fallback when docling is installed; otherwise the anydoc output is kept
+    # and the document gets extra_info["parse_warnings"]. ~30 ms per scanned MB.
+    PDF_TRUST_CHECK: bool = True
+    # Rewrite dot-leader / whitespace-aligned table runs in anydoc's PDF
+    # markdown into GFM tables (application/parser/file/tableize.py). Off by
+    # default: it rewrites content on a heuristic (>=3 uniform label+numbers
+    # lines) validated only on a small corpus so far.
+    ANYDOC_TABLEIZE: bool = False
+    # OCR engine used when OCR is on (OCR_ENABLED / OCR_ATTACHMENTS_ENABLED).
+    # Benched 2026-08 on EN/ZH/table/degraded scans (docs/Guides/ocr has the
+    # menu):
+    #   tesseract — recommended: best classic-engine accuracy (perfect EN word
+    #     recall, 0.000 bilingual CER, 100% table cells), ~35 MB, CPU-only.
+    #     Needs the system binary + language packs: an optional install like
+    #     every OCR dependency (build with INSTALL_TESSERACT=true, or apt/brew
+    #     install tesseract-ocr for a local run). Both backends.
+    #   deepseek — DeepSeek-OCR against an Ollama/vLLM endpoint
+    #     (OCR_DEEPSEEK_*). Best table/CJK quality; the worker stays light
+    #     (no layout models) but each page costs seconds on the model server.
+    #     Both backends.
+    #   auto — docling's pick: ocrmac on macOS (excellent), rapidocr on Linux
+    #     (silently shreds some long text lines — avoid as a server default).
+    #   ocrmac | rapidocr — force one of those.
+    # auto/ocrmac/rapidocr exist only inside docling; the native backend runs
+    # tesseract for them. An engine that is not installed degrades (docling:
+    # to "auto") with a warning instead of failing the parse.
+    OCR_ENGINE: str = "tesseract"
+    # Tesseract language packs, "+"-separated (e.g. "eng+chi_sim+deu"). Other
+    # engines keep their own defaults — their language codes differ.
+    OCR_LANGS: str = "eng"
+    OCR_DEEPSEEK_URL: str = "http://localhost:11434/v1/chat/completions"
+    OCR_DEEPSEEK_MODEL: str = "deepseek-ocr:3b"
+    # Seconds allowed per page request to the DeepSeek endpoint, on both
+    # backends (native sends pages one at a time; docling's VLM pipeline
+    # keeps its own concurrency). A 3B model on a laptop needs minutes; a
+    # vLLM GPU deployment, seconds.
+    OCR_DEEPSEEK_TIMEOUT: float = 300.0
+    # Native backend only: resolution at which pages without a text layer are
+    # rendered before OCR. 200 suits tesseract; clamped to 72-600.
+    OCR_RENDER_DPI: int = 200
+    # Chars-per-page floor below which an OCR'd PDF/image parse is treated as an OCR
+    # dropout (long-running docling workers were observed returning zero characters for
+    # every scanned page after a long scanned PDF, with no error) rather than as content.
+    # docling retries once on a fresh full-page-OCR converter; both backends then fail
+    # loudly instead of indexing an empty document. 0 disables the guard.
+    OCR_MIN_CHARS_PER_PAGE: int = Field(
+        default=20, validation_alias=AliasChoices("OCR_MIN_CHARS_PER_PAGE", "DOCLING_OCR_MIN_CHARS_PER_PAGE")
+    )
+    # Read PDF *attachments* via their embedded text layer (pypdfium2) instead
+    # of docling, falling back to docling when there is no text layer to read.
+    # Attachments go into a prompt, so docling's structural markdown earns far
+    # less than the tens of seconds per file it costs; source ingestion is
+    # unaffected and always uses docling, because chunking and retrieval do
+    # depend on that structure.
     ATTACHMENT_PDF_TEXT_FAST_PATH: bool = True
     # Median chars per sampled page below which a PDF is treated as a scan and handed to docling.
     # Measured on real uploads: scans at 0-17 chars/page, text-layer documents at 433-6834.

application/parser/document_reader.py

@@ -1,10 +1,11 @@
 """In-process document parsing for the ``read_document`` tool, run on the Celery parsing worker.
 
 ``parse_document_bytes`` turns untrusted document bytes into a bounded, shaped
-result (markdown/text/structured/chunks) using the BACKEND parsers (Docling by
-default). It applies the same untrusted-content safeguards as uploads — an
-extension whitelist, a byte cap, ``safe_filename`` staging into a temp file, and
-temp cleanup — so a hostile filename or document is treated as inert data.
+result (markdown/text/structured/chunks) using the BACKEND parsers (the engine
+selected by ``DOC_PARSER_ENGINE``; ``structured`` output always needs Docling).
+It applies the same untrusted-content safeguards as uploads — an extension
+whitelist, a byte cap, ``safe_filename`` staging into a temp file, and temp
+cleanup — so a hostile filename or document is treated as inert data.
 """
 
 from __future__ import annotations
@@ -40,7 +41,7 @@ _MAX_PAGE_SELECTOR_TOKENS = 20_000
 
 _VALID_OUTPUTS = ("markdown", "text", "structured", "chunks")
 _VALID_OCR = ("auto", "on", "off")
-_VALID_ENGINES = ("auto", "docling", "fast")
+_VALID_ENGINES = ("auto", "docling", "anydoc", "fast")
 
 
 def truncate_text_head_tail(text: str, max_bytes: Optional[int] = None) -> str:
@@ -102,7 +103,13 @@ def _max_input_bytes() -> int:
     return int(getattr(settings, "SANDBOX_MAX_INPUT_BYTES", 25 * 1024 * 1024))
 
 
-_ZIP_CONTAINER_EXTENSIONS = frozenset({".docx", ".xlsx", ".pptx", ".epub"})
+# Every zip-packaged format a parser map can route: OOXML and its macro/
+# binary variants, OpenDocument, EPUB. Chat attachments have no extension
+# whitelist, so the anydoc engine can receive any of these.
+_ZIP_CONTAINER_EXTENSIONS = frozenset({
+    ".docx", ".docm", ".xlsx", ".xlsm", ".xlsb", ".pptx", ".pptm", ".ppsx", ".ppsm",
+    ".odt", ".ods", ".odp", ".epub",
+})
 
 
 def _zip_bomb_reason(source: Union[bytes, str, Path], suffix: str) -> Optional[str]:
@@ -162,7 +169,15 @@ def _resolve_ocr_enabled(ocr: str) -> bool:
         return True
     if ocr == "off":
         return False
-    return bool(getattr(settings, "DOCLING_OCR_ENABLED", False))
+    return bool(getattr(settings, "OCR_ENABLED", False))
+
+
+def _effective_engine(engine: str) -> str:
+    """Resolve ``auto`` to the server's ``DOC_PARSER_ENGINE``; other values pass through."""
+    if engine != "auto":
+        return engine
+    configured = getattr(settings, "DOC_PARSER_ENGINE", None) or "anydoc"
+    return str(configured).strip().lower()
 
 
 def _pick_parser(suffix: str, *, ocr_enabled: bool, engine: str):
@@ -171,25 +186,44 @@ def _pick_parser(suffix: str, *, ocr_enabled: bool, engine: str):
         legacy = _legacy_parser_for(suffix)
         if legacy is not None:
             return legacy
-    extractor = get_default_file_extractor(ocr_enabled=ocr_enabled)
+        from application.parser.file.anydoc_parser import ANYDOC_GAINED_SUFFIXES
+
+        if suffix in ANYDOC_GAINED_SUFFIXES:
+            # anydoc-only office formats have no legacy parser. Falling through
+            # would run the configured engine the caller explicitly opted out
+            # of; images/audio/.txt still fall through (no legacy alternative).
+            return None
+    if engine in ("docling", "anydoc"):
+        extractor = get_default_file_extractor(ocr_enabled=ocr_enabled, engine=engine)
+    else:
+        extractor = get_default_file_extractor(ocr_enabled=ocr_enabled)
     return extractor.get(suffix)
 
 
 def _legacy_parser_for(suffix: str):
     """Return a non-Docling parser for ``suffix`` (the ``fast`` engine), or None."""
     from application.parser.file.docs_parser import DocxParser, PDFParser
+    from application.parser.file.epub_parser import EpubParser
     from application.parser.file.html_parser import HTMLParser
+    from application.parser.file.json_parser import JSONParser
     from application.parser.file.markdown_parser import MarkdownParser
+    from application.parser.file.pptx_parser import PPTXParser
+    from application.parser.file.rst_parser import RstParser
     from application.parser.file.tabular_parser import ExcelParser, PandasCSVParser
 
     legacy = {
         ".pdf": PDFParser,
         ".docx": DocxParser,
+        ".pptx": PPTXParser,
         ".csv": PandasCSVParser,
         ".xlsx": ExcelParser,
         ".html": HTMLParser,
+        ".xhtml": HTMLParser,
+        ".epub": EpubParser,
         ".md": MarkdownParser,
         ".mdx": MarkdownParser,
+        ".rst": RstParser,
+        ".json": JSONParser,
     }
     cls = legacy.get(suffix)
     return cls() if cls is not None else None
@@ -205,6 +239,17 @@ def _parse_to_text(parser: Any, path: Path) -> str:
     return str(parsed)
 
 
+def _is_native_ocr_parser(parser: Any) -> bool:
+    """True for the native OCR PDF parser (OCR_BACKEND=native): a docling table pass would OCR the scan again."""
+    if parser is None:
+        return False
+    try:
+        from application.parser.file.ocr_parser import NativeOcrPdfParser
+    except Exception:
+        return False
+    return isinstance(parser, NativeOcrPdfParser)
+
+
 def _is_docling_parser(parser: Any) -> bool:
     """True when ``parser`` is Docling-backed (collecting tables would otherwise re-convert)."""
     if parser is None:
@@ -491,7 +536,29 @@ def _shape(
         }
 
     parser = _pick_parser(suffix, ocr_enabled=ocr_enabled, engine=engine)
-    wants_tables = include_tables and engine != "fast" and output != "chunks"
+    # Under OCR_BACKEND=native with the docling engine the PDF parser is the
+    # native OCR parser wrapping a DoclingParser as its text_parser. A PDF
+    # with a text layer on every page is handed to that parser untouched, so
+    # when tables are wanted run it directly: the single docling conversion
+    # below then yields both the markdown and the tables. Scans and mixed
+    # documents keep the native parser (and the exclusion just below).
+    if include_tables and output != "chunks" and _is_native_ocr_parser(parser):
+        delegate = parser.text_layer_delegate(path)
+        if _is_docling_parser(delegate):
+            parser = delegate
+    # Tables come from a Docling conversion, so they are only collected when
+    # a Docling parser is actually in play: under anydoc/fast a table pass
+    # would be a second, full docling conversion that costs more than the
+    # whole read, and under OCR_BACKEND=native the PDF parser is the native
+    # OCR parser (docling only as its text_parser), where a vanilla
+    # DocumentConverter pass would OCR the scan a second time with docling's
+    # default engine, ignoring OCR_ENGINE.
+    wants_tables = (
+        include_tables
+        and output != "chunks"
+        and _effective_engine(engine) == "docling"
+        and not _is_native_ocr_parser(parser)
+    )
 
     # A Docling-backed parser already converts the whole document to produce its text.
     # When tables are also requested, reuse that single conversion for both the markdown
@@ -513,7 +580,21 @@ def _shape(
 
     if parser is None:
         # A whitelisted extension with no dedicated parser (e.g. .txt) reads as plain
-        # text, matching SimpleDirectoryReader's standard-read fallback.
+        # text, matching SimpleDirectoryReader's standard-read fallback. Binary
+        # office formats that only anydoc reads must not: without anydoc they
+        # would come back as OLE/zip bytes decoded as text.
+        from application.parser.file.anydoc_parser import ANYDOC_GAINED_SUFFIXES
+
+        if suffix in ANYDOC_GAINED_SUFFIXES:
+            if engine == "fast":
+                return {
+                    "error": f"The fast engine has no parser for {suffix} files; "
+                    "use engine='anydoc' or 'auto'."
+                }
+            return {
+                "error": f"No parser is available for {suffix} files: firecrawl-anydoc "
+                "is not installed (pip install firecrawl-anydoc)"
+            }
         text = path.read_text(errors="ignore")
     else:
         text = _parse_to_text(parser, path)

application/parser/file/anydoc_parser.py

@@ -0,0 +1,417 @@
+"""anydoc parser.
+
+Converts documents to GitHub-Flavored Markdown with `firecrawl-anydoc
+<https://github.com/firecrawl/anydoc>`_: a single Rust extension with no ML
+models and no Python dependencies. Measured against docling on a 20-document
+corpus it was 20-160x faster per file (1.2 s vs 193 s on a 150-page annual
+report), imported in 6 ms instead of 3.9 s, and peaked at 107 MB RSS instead
+of 2.3 GB, with equivalent output on clean office files, CSV and text-layer
+PDFs.
+
+anydoc never OCRs. It *detects* a scanned or image-only PDF and raises
+``UnsupportedError`` ("OCR is required"), which is the routing point: a
+parser given a ``fallback_parser`` (docling when installed, the native OCR
+parser when OCR is on without docling, otherwise the legacy parser for the
+suffix) delegates there; without one the file fails loudly as
+``DocumentParseError`` rather than being stored empty.
+"""
+import logging
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple, Union
+
+from application.core.settings import settings
+from application.parser.file.base_parser import (
+    BaseParser,
+    DocumentParseError,
+    delegate_parse,
+    module_available,
+)
+
+logger = logging.getLogger(__name__)
+
+# A fallback parse (scanned-PDF delegation or trust-check re-parse) yielding
+# fewer stripped characters than this is an empty document in the making,
+# not content.
+_MIN_SCAN_FALLBACK_CHARS = 50
+
+# Suffixes anydoc 0.2.3 converts, verified with ``anydoc.format_from_extension``.
+# ``.epub`` is deliberately absent: it stays on ``EpubParser``.
+ANYDOC_SUFFIXES: Tuple[str, ...] = (
+    # Word processing
+    ".pdf",
+    ".docx",
+    ".docm",
+    ".doc",
+    ".odt",
+    ".rtf",
+    # Presentations
+    ".pptx",
+    ".pptm",
+    ".ppsx",
+    ".ppsm",
+    ".ppt",
+    ".pps",
+    ".pot",
+    ".odp",
+    # Spreadsheets / tabular
+    ".xlsx",
+    ".xlsm",
+    ".xlsb",
+    ".xls",
+    ".ods",
+    ".csv",
+)
+
+
+# The five formats every engine has its own parser for (docling / legacy);
+# under the anydoc engine they get that parser as the fallback. Everything
+# else in ``ANYDOC_SUFFIXES`` is anydoc-only ("gained") and is mapped to a
+# fallback-less ``AnydocParser`` in every parser map.
+_CORE_SUFFIXES = frozenset({".pdf", ".docx", ".pptx", ".xlsx", ".csv"})
+
+# Spreadsheets anydoc may refuse on its fixed, non-configurable limits
+# (2M XML nodes per part, 128 MiB decompressed per part — a ~265k-row sheet
+# trips them) that the plain tabular parsers read fine in a few hundred MB.
+# For these a ResourceLimitError is a reason to delegate, not to fail.
+_TABULAR_SUFFIXES = frozenset({".csv", ".xlsx", ".xlsm", ".xlsb", ".xls", ".ods"})
+ANYDOC_GAINED_SUFFIXES: Tuple[str, ...] = tuple(
+    suffix for suffix in ANYDOC_SUFFIXES if suffix not in _CORE_SUFFIXES
+)
+
+
+def anydoc_available() -> bool:
+    """Whether the ``anydoc`` extension can be imported."""
+    return module_available("anydoc")
+
+
+def _result_chars(result: Union[str, List[str]]) -> int:
+    """Stripped character count of a parser result (str or list of rows)."""
+    if isinstance(result, list):
+        return sum(len(str(part).strip()) for part in result)
+    return len(str(result).strip())
+
+
+def _is_docling_backed(parser: Optional[BaseParser]) -> bool:
+    """True when ``parser`` is a docling parser (worth a trust-check re-parse).
+
+    Looks through ``NativeOcrPdfParser``, which under ``OCR_BACKEND=native``
+    wraps the docling PDF parser as its ``text_parser`` and hands text-layer
+    PDFs — the only kind the trust check flags — straight to it.
+    """
+    if parser is None:
+        return False
+    try:
+        from application.parser.file.docling_parser import DoclingParser
+    except ImportError:
+        return False
+    if isinstance(parser, DoclingParser):
+        return True
+    return isinstance(getattr(parser, "text_parser", None), DoclingParser)
+
+
+class AnydocParser(BaseParser):
+    """Markdown conversion via ``anydoc.to_markdown`` with a typed fallback.
+
+    Attributes:
+        fallback_parser: Parser used when anydoc cannot convert the file
+            (scanned PDF, unsupported or malformed input). ``None`` means
+            such files raise ``DocumentParseError``.
+        last_engine: Name of the engine that produced the most recent parse:
+            ``"anydoc"``, or the fallback parser's ``last_engine`` / class
+            name when it was delegated to. Mirrors ``PdfiumTextParser`` so
+            the attachment worker records what actually ran.
+    """
+
+    def __init__(
+        self,
+        fallback_parser: Optional[BaseParser] = None,
+        parser_config: Optional[Dict] = None,
+    ) -> None:
+        super().__init__(parser_config)
+        self.fallback_parser = fallback_parser
+        self.last_engine: Optional[str] = None
+        # (path, problems) from the most recent parse whose output the trust
+        # check flagged but kept — surfaced via ``get_file_metadata`` as
+        # ``parse_warnings`` so the document carries its own caveat.
+        self._last_warnings: Optional[Tuple[Path, List[str]]] = None
+        # (path, count) of scanned pages OCR'd into the most recent parse.
+        self._last_ocr_pages: Optional[Tuple[Path, int]] = None
+
+    def _init_parser(self) -> Dict:
+        # Import for real rather than trusting ``find_spec``: a wheel whose
+        # native extension fails to load (glibc/arch mismatch) has a spec but
+        # no importable module, and that must fail here — where the ingest
+        # task treats it as a setup error — not as a bare ImportError from
+        # ``parse_file`` mid-batch, which ``load_data`` does not catch.
+        try:
+            import anydoc  # noqa: F401
+        except ImportError as exc:
+            raise ImportError(
+                "firecrawl-anydoc is required for AnydocParser. "
+                "Install it with: pip install firecrawl-anydoc"
+            ) from exc
+        fallback = self.fallback_parser
+        return {"fallback_parser": type(fallback).__name__ if fallback else None}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Convert ``file`` to Markdown.
+
+        Args:
+            file: Path to the document.
+            errors: Decoding error policy; anydoc decodes internally, so this
+                is only forwarded to the fallback parser.
+
+        Returns:
+            The document as Markdown.
+
+        Raises:
+            DocumentParseError: When neither anydoc nor the fallback could
+                produce text for the file, or anydoc hit a resource limit.
+        """
+        import anydoc
+
+        path = Path(file)
+        self._last_warnings = None
+        self._last_ocr_pages = None
+        try:
+            content = anydoc.to_markdown(str(path))
+        except anydoc.ResourceLimitError as exc:
+            # anydoc refused because the file is too expensive (declared
+            # decompression size, nesting depth, node count). For office
+            # documents a heavier engine would spend exactly what was just
+            # refused, so this is terminal. Spreadsheets are the exception:
+            # anydoc's cell/node limits sit below what ordinary business
+            # exports reach, and the tabular fallback (docling's gate hands
+            # oversized sheets to pandas/openpyxl) is lighter, not heavier.
+            if path.suffix.lower() in _TABULAR_SUFFIXES and self.fallback_parser is not None:
+                return self._delegate(path, errors, f"{type(exc).__name__}: {exc}")
+            self.last_engine = None
+            raise DocumentParseError(
+                f"Failed to parse {path.name}: {type(exc).__name__}: {exc}"
+            ) from exc
+        except anydoc.ConvertError as exc:
+            # Typed, per-document failures another engine may still handle:
+            # UnsupportedError (scanned PDF / unknown format), MalformedError,
+            # EncryptedError, MissingPartError.
+            ocr_needed = isinstance(exc, anydoc.UnsupportedError) and "OCR" in str(exc)
+            return self._delegate(
+                path, errors, f"{type(exc).__name__}: {exc}", ocr_needed=ocr_needed
+            )
+        except OSError as exc:
+            raise DocumentParseError(
+                f"Failed to parse {path.name}: the file could not be read."
+            ) from exc
+        except Exception as exc:
+            logger.error(f"anydoc failed on {path.name}: {exc}", exc_info=True)
+            raise DocumentParseError(
+                f"Failed to parse {path.name} with anydoc: {exc}"
+            ) from exc
+
+        if not content or not content.strip():
+            return self._delegate(path, errors, "anydoc produced no text")
+
+        self.last_engine = "anydoc"
+        if path.suffix.lower() == ".pdf":
+            content = self._finish_pdf(path, content, errors)
+        return content
+
+    def _finish_pdf(self, path: Path, content: str, errors: str) -> Union[str, List[str]]:
+        """Post-process a successful anydoc PDF conversion.
+
+        Three PDF-only steps:
+
+        1. Trust check (``PDF_TRUST_CHECK``): the two classes where anydoc
+           drops text *silently* — Type0 fonts without ToUnicode, and
+           CJK-declaring PDFs with CJK-less output. A flagged file re-parses
+           on the docling fallback when one is wired; otherwise the anydoc
+           output is kept and the problems ride along as ``parse_warnings``
+           metadata.
+        2. Mixed documents: anydoc only refuses a PDF when *every* page lacks
+           text, so a text document with scanned pages in it converts fine
+           and silently loses those pages. When the fallback parser can OCR
+           (it is wired with OCR on), the pages with no text layer are OCR'd
+           through it and appended (``_ocr_scanned_pages``).
+        3. ``ANYDOC_TABLEIZE`` (off by default): rewrite dot-leader /
+           whitespace table runs as GFM tables. Never applied to a docling
+           re-parse — docling emits real tables already.
+        """
+        problems = self._trust_problems(path, content)
+        if problems:
+            rerouted = self._reroute_flagged(path, errors, problems)
+            if rerouted is not None:
+                return rerouted
+            self._last_warnings = (path, problems)
+            logger.warning(
+                f"anydoc output for {path.name} failed the PDF trust check "
+                f"({'; '.join(problems)}); keeping it with parse_warnings"
+            )
+        content = self._ocr_scanned_pages(path, content)
+        if settings.ANYDOC_TABLEIZE:
+            from application.parser.file.tableize import tableize
+
+            content = tableize(content)
+        return content
+
+    def _trust_problems(self, path: Path, content: str) -> List[str]:
+        """Trust-check findings for ``content``; [] when disabled, clean, or the check errors."""
+        if not settings.PDF_TRUST_CHECK:
+            return []
+        try:
+            from application.parser.file.pdf_trust import verify_pdf_file
+
+            return verify_pdf_file(path, content)
+        except Exception:
+            # The check exists to catch silent loss; it must never turn a
+            # successful parse into a failure.
+            logger.warning(
+                f"PDF trust check errored on {path.name}; trusting the output",
+                exc_info=True,
+            )
+            return []
+
+    def _reroute_flagged(
+        self, path: Path, errors: str, problems: List[str]
+    ) -> Optional[Union[str, List[str]]]:
+        """Re-parse a trust-flagged PDF with a docling-backed fallback.
+
+        Returns the fallback's output, or None when there is no docling
+        fallback, it fails, or it comes back near-empty — the caller then
+        keeps the anydoc output.
+        Only docling is worth the re-parse: it ships Adobe's predefined
+        CMaps, which is exactly what the flagged font class needs; the
+        legacy pypdf parser does not.
+        """
+        fallback = self.fallback_parser
+        if not _is_docling_backed(fallback):
+            return None
+        logger.warning(
+            f"anydoc output for {path.name} failed the PDF trust check "
+            f"({'; '.join(problems)}); re-parsing with {type(fallback).__name__}"
+        )
+        try:
+            result = delegate_parse(fallback, path, errors)
+        except DocumentParseError:
+            logger.warning(
+                f"docling re-parse of trust-flagged {path.name} failed; "
+                "keeping the anydoc output",
+                exc_info=True,
+            )
+            return None
+        if _result_chars(result) < _MIN_SCAN_FALLBACK_CHARS:
+            # A docling pipeline dropout ('' / '<!-- image -->') returns
+            # without raising; adopting it would swap anydoc's real text for
+            # an empty document.
+            logger.warning(
+                f"docling re-parse of trust-flagged {path.name} returned almost "
+                "no text; keeping the anydoc output"
+            )
+            return None
+        self.last_engine = getattr(fallback, "last_engine", None) or type(fallback).__name__
+        return result
+
+    def _ocr_scanned_pages(self, path: Path, content: str) -> str:
+        """Append OCR text for the pages of ``path`` that have no text layer.
+
+        Runs only when the fallback parser both has OCR on and exposes
+        ``ocr_pages`` (the native OCR PDF parser, or the docling PDF parser).
+        With OCR off the fallback is the plain pypdf parser and mixed
+        documents keep today's behaviour: text pages only. Pages are OCR'd
+        one at a time so a failure on one page costs that page, not the OCR
+        of every other; the anydoc text is always kept — a partial document
+        beats a rejected upload, and the loud-failure path stays reserved
+        for fully scanned files. anydoc's PDF markdown carries no page
+        separators, so the OCR'd pages are appended after the text pages
+        rather than spliced into position.
+        """
+        fallback = self.fallback_parser
+        if not (getattr(fallback, "ocr_enabled", False) and hasattr(fallback, "ocr_pages")):
+            return content
+        from application.parser.file.ocr_parser import scanned_page_indices
+
+        indices = scanned_page_indices(path)
+        if not indices:
+            return content
+        logger.info(
+            "%s has %d page(s) without a text layer among its text pages; OCR-ing them with %s",
+            path.name,
+            len(indices),
+            type(fallback).__name__,
+        )
+        try:
+            if not fallback.parser_config_set:
+                fallback.init_parser()
+        except Exception:  # noqa: BLE001 - keep the text pages rather than fail the file
+            logger.warning(
+                "OCR of the scanned pages of %s failed; indexing the text pages only",
+                path.name,
+                exc_info=True,
+            )
+            return content
+        extra: List[str] = []
+        for index in indices:
+            try:
+                text = fallback.ocr_pages(path, [index]).get(index, "")
+            except Exception:  # noqa: BLE001 - one page's failure must not drop the others
+                logger.warning(
+                    "OCR of page %d of %s failed; skipping that page",
+                    index + 1,
+                    path.name,
+                    exc_info=True,
+                )
+                continue
+            if text and text.strip():
+                extra.append(text)
+        if not extra:
+            return content
+        self._last_ocr_pages = (path, len(extra))
+        return content.rstrip() + "\n\n" + "\n\n".join(extra) + "\n"
+
+    def get_file_metadata(self, file: Path) -> Dict:
+        """Surface trust-check findings and OCR'd page counts for the file just parsed."""
+        metadata: Dict = {}
+        last = self._last_warnings
+        if last is not None and last[0] == Path(file):
+            metadata["parse_warnings"] = list(last[1])
+        ocr = self._last_ocr_pages
+        if ocr is not None and ocr[0] == Path(file):
+            metadata["ocr_pages"] = ocr[1]
+        return metadata
+
+    def _delegate(
+        self, path: Path, errors: str, reason: str, ocr_needed: bool = False
+    ) -> Union[str, List[str]]:
+        """Hand ``path`` to the fallback parser, or fail loudly without one.
+
+        ``ocr_needed`` marks anydoc's scanned-PDF refusal ("OCR is required").
+        The fallback still runs first — docling extracts text layers anydoc
+        refuses (seen on degenerate CJK text layers) even with OCR off — but
+        when it comes back near-empty the parse fails loudly, telling the
+        user how to get OCR, instead of silently storing an empty document
+        for a scan.
+        """
+        fallback = self.fallback_parser
+        if fallback is None:
+            self.last_engine = None
+            raise DocumentParseError(f"Failed to parse {path.name}: {reason}")
+        logger.warning(
+            f"anydoc could not convert {path.name} ({reason}); "
+            f"falling back to {type(fallback).__name__}"
+        )
+        result = delegate_parse(fallback, path, errors)
+        if ocr_needed and _result_chars(result) < _MIN_SCAN_FALLBACK_CHARS:
+            self.last_engine = None
+            if getattr(fallback, "ocr_enabled", False):
+                hint = " even with OCR enabled."
+            else:
+                hint = (
+                    ". Enable OCR to ingest scans: set OCR_ENABLED=true (and "
+                    "OCR_ATTACHMENTS_ENABLED for attachments). OCR runs on the "
+                    "tesseract binary or a DeepSeek-OCR endpoint (OCR_ENGINE), or "
+                    "through the optional docling extra when installed."
+                )
+            raise DocumentParseError(
+                f"{path.name} appears to be a scanned PDF (no text layer), and "
+                f"{type(fallback).__name__} extracted almost nothing{hint}"
+            )
+        self.last_engine = getattr(fallback, "last_engine", None) or type(fallback).__name__
+        return result

application/parser/file/base_parser.py

@@ -1,9 +1,14 @@
 """Base parser and config class."""
 
+import importlib.util
+import logging
+import sys
 from abc import abstractmethod
 from pathlib import Path
 from typing import Dict, List, Optional, Union
 
+logger = logging.getLogger(__name__)
+
 
 class DocumentParseError(Exception):
     """A file could not be converted to text.
@@ -57,3 +62,72 @@ class BaseParser:
         """Return parser-specific metadata for the most recently parsed file."""
         _ = file
         return {}
+
+
+def module_available(name: str) -> bool:
+    """Whether top-level module ``name`` can be imported, without importing it.
+
+    Cheap enough to call while building a parser map. Honors ``sys.modules``
+    first so tests that stub a module in (or block one out with ``None``)
+    are seen the same way the import system would see them; a stub with no
+    ``__spec__`` counts as present rather than raising.
+
+    Args:
+        name: Top-level module name, e.g. ``"docling"``.
+
+    Returns:
+        True when an import of ``name`` would succeed.
+    """
+    if name in sys.modules:
+        return sys.modules[name] is not None
+    try:
+        return importlib.util.find_spec(name) is not None
+    except (ImportError, ValueError):
+        return False
+
+
+def delegate_parse(
+    parser: "BaseParser", file: Path, errors: str
+) -> Union[str, List[str]]:
+    """Run a fallback parser, normalizing its failures to ``DocumentParseError``.
+
+    Used wherever one parser hands a file to another (docling to the plain
+    tabular parsers for oversized sheets, anydoc to docling/legacy parsers
+    for files it cannot convert). Wrapping matters twice over: the Celery
+    tasks that own these paths list ``DocumentParseError`` in
+    ``dont_autoretry_for``, so a deterministic content error fails once
+    instead of retrying four times; and ``SimpleDirectoryReader.load_data``
+    catches only ``DocumentParseError``, so one poison file is skipped into
+    ``failed_files`` instead of aborting an entire multi-file ingest.
+
+    Args:
+        parser: The parser to delegate to; initialized here if needed.
+        file: Path to the file being parsed.
+        errors: Decoding error policy, forwarded to the parser.
+
+    Returns:
+        The parsed text, or list of row strings.
+
+    Raises:
+        DocumentParseError: If the fallback parser fails for any reason.
+    """
+    try:
+        if not parser.parser_config_set:
+            parser.init_parser()
+        return parser.parse_file(file, errors)
+    except DocumentParseError:
+        raise
+    except ImportError:
+        # A fallback whose dependency is missing is a deployment problem, not
+        # a bad document: let it reach the ingest task's setup-error path
+        # instead of blaming every file with "could not be read".
+        raise
+    except Exception as e:
+        logger.error(
+            f"Fallback parse of {Path(file).name} with "
+            f"{type(parser).__name__} failed: {e}",
+            exc_info=True,
+        )
+        raise DocumentParseError(
+            f"Failed to parse {Path(file).name}: the file could not be read."
+        ) from e

application/parser/file/bulk.py

@@ -4,7 +4,7 @@ from pathlib import Path
 from typing import Callable, Dict, List, Optional, Tuple, Union
 
 from application.parser.file.base import BaseReader
-from application.parser.file.base_parser import BaseParser, DocumentParseError
+from application.parser.file.base_parser import BaseParser, DocumentParseError, module_available
 from application.parser.file.docs_parser import DocxParser, PDFParser
 from application.parser.file.epub_parser import EpubParser
 from application.parser.file.html_parser import HTMLParser
@@ -49,28 +49,140 @@ def _wrap_pdf_fast_path(pdf_parser: BaseParser) -> BaseParser:
     )
 
 
-def get_default_file_extractor(
-    ocr_enabled: Optional[bool] = None,
-    pdf_text_fast_path: bool = False,
-) -> Dict[str, BaseParser]:
-    """Get the default file extractor.
+def _gained_format_entries() -> Dict[str, BaseParser]:
+    """Anydoc-only formats (legacy/macro Office, OpenDocument, RTF) for every map.
 
-    Uses docling parsers by default for advanced document processing.
-    Falls back to standard parsers if docling is not installed.
+    anydoc is a core dependency, so these suffixes are parseable under both
+    engines. When anydoc is somehow missing the entries are omitted and
+    ``SimpleDirectoryReader.load_data`` rejects such files with a
+    ``DocumentParseError`` naming the install, rather than reading OLE/zip
+    bytes as text.
+    """
+    from application.parser.file.anydoc_parser import (
+        ANYDOC_GAINED_SUFFIXES,
+        AnydocParser,
+        anydoc_available,
+    )
+
+    if not anydoc_available():
+        return {}
+    return {suffix: AnydocParser() for suffix in ANYDOC_GAINED_SUFFIXES}
+
+
+def _native_ocr_parsers(
+    text_parser: Optional[BaseParser] = None,
+) -> Optional[Tuple[BaseParser, Callable[[], BaseParser]]]:
+    """PDF parser and image-parser factory for the native OCR backend.
 
     Args:
-        ocr_enabled: Enable OCR in the docling parsers. Defaults to
-            ``settings.DOCLING_OCR_ENABLED``.
-        pdf_text_fast_path: Read PDFs via their embedded text layer
-            (pypdfium2) instead of docling, falling back to docling for scans.
-            Off by default: it trades docling's structural markdown for speed,
-            which suits attachments (read into a prompt) but not source
-            ingestion (chunked and embedded for retrieval).
+        text_parser: Parser for PDFs whose every page has a text layer (the
+            docling PDF parser under the docling engine); None reads text
+            layers with pypdfium2.
+
+    Returns:
+        ``(pdf_parser, image_parser_factory)``, or None when pypdfium2 or
+        Pillow is missing — both are core dependencies, so that is a broken
+        install, logged as such.
+    """
+    from application.parser.file.ocr_parser import (
+        NativeOcrImageParser,
+        NativeOcrPdfParser,
+        native_ocr_available,
+    )
+
+    if not native_ocr_available():
+        logging.error(
+            "OCR is enabled but pypdfium2/Pillow are not importable, so the native "
+            "OCR backend cannot run; scanned PDFs and images will not be OCR'd"
+        )
+        return None
+    return NativeOcrPdfParser(text_parser=text_parser), NativeOcrImageParser
+
+
+def _image_entries(factory: Callable[[], BaseParser], suffixes) -> Dict[str, BaseParser]:
+    """One parser instance per image suffix."""
+    return {suffix: factory() for suffix in suffixes}
+
+
+_LEGACY_IMAGE_SUFFIXES = (".png", ".jpg", ".jpeg")
+
+
+def _legacy_file_extractor(pdf_text_fast_path: bool = False, ocr_enabled: bool = False) -> Dict[str, BaseParser]:
+    """Parser map that needs neither docling nor anydoc.
+
+    Args:
+        pdf_text_fast_path: Put the pypdfium2 text-layer parser in front of
+            the PDF parser.
+        ocr_enabled: Use the native OCR parsers (tesseract / DeepSeek-OCR via
+            pypdfium2 + Pillow) for PDFs and images instead of the plain
+            pypdf / remote-service parsers.
+
+    Returns:
+        Dict[str, BaseParser]: Parser keyed by lower-case file suffix.
+    """
+    from application.parser.file.ocr_parser import IMAGE_SUFFIXES
+
+    pdf_parser: BaseParser = PDFParser()
+    images: Dict[str, BaseParser] = _image_entries(ImageParser, _LEGACY_IMAGE_SUFFIXES)
+    if ocr_enabled:
+        native = _native_ocr_parsers()
+        if native is not None:
+            pdf_parser, image_factory = native
+            images = _image_entries(image_factory, sorted(IMAGE_SUFFIXES))
+    if pdf_text_fast_path:
+        pdf_parser = _wrap_pdf_fast_path(pdf_parser)
+    return {
+        ".pdf": pdf_parser,
+        ".docx": DocxParser(),
+        ".csv": PandasCSVParser(),
+        ".xlsx": ExcelParser(),
+        ".epub": EpubParser(),
+        ".md": MarkdownParser(),
+        ".rst": RstParser(),
+        ".html": HTMLParser(),
+        ".xhtml": HTMLParser(),
+        ".mdx": MarkdownParser(),
+        ".json": JSONParser(),
+        ".pptx": PPTXParser(),
+        **images,
+        **_build_audio_parser_mapping(),
+        **_gained_format_entries(),
+    }
+
+
+def _docling_file_extractor(
+    ocr_enabled: bool,
+    pdf_text_fast_path: bool = False,
+    *,
+    missing_log_level: int = logging.WARNING,
+) -> Dict[str, BaseParser]:
+    """docling parser map, or the legacy map when docling is not installed.
+
+    With OCR on, ``OCR_BACKEND`` decides who OCRs: docling's own hybrid
+    pipeline, or the native parsers (``ocr_parser.py``) with the docling PDF
+    parser kept — OCR off — for PDFs that have a text layer on every page.
+    Without docling installed the legacy map takes over, itself using the
+    native OCR parsers when OCR is on.
+
+    The availability check has to happen here, not at parse time:
+    ``DoclingParser`` only imports docling inside ``_init_parser``, and the
+    ``ImportError`` that raises is not a ``DocumentParseError``, so a map
+    that handed out docling parsers on a docling-less install would abort
+    the whole ingest on its first file instead of degrading.
+
+    Args:
+        ocr_enabled: Enable OCR in the docling PDF/image parsers.
+        pdf_text_fast_path: Put the pypdfium2 text-layer parser in front of
+            the PDF parser (attachments only; see ``get_default_file_extractor``).
+        missing_log_level: Level at which a missing docling install is logged;
+            informational when docling is only the *fallback* engine.
 
     Returns:
         Dict[str, BaseParser]: Parser keyed by lower-case file suffix.
     """
     try:
+        if not module_available("docling"):
+            raise ImportError("docling is not installed")
         from application.parser.file.docling_parser import (
             DoclingPDFParser,
             DoclingDocxParser,
@@ -83,71 +195,160 @@ def get_default_file_extractor(
             DoclingVTTParser,
             DoclingXMLParser,
         )
-        if ocr_enabled is None:
-            ocr_enabled = settings.DOCLING_OCR_ENABLED
-        pdf_parser: BaseParser = DoclingPDFParser(ocr_enabled=ocr_enabled)
-        if pdf_text_fast_path:
-            pdf_parser = _wrap_pdf_fast_path(pdf_parser)
-        return {
-            # Documents
-            ".pdf": pdf_parser,
-            ".docx": DoclingDocxParser(),
-            ".pptx": DoclingPPTXParser(),
-            ".xlsx": DoclingXLSXParser(),
-            # Web formats
-            ".html": DoclingHTMLParser(),
-            ".xhtml": DoclingHTMLParser(),
-            # Data formats
-            ".csv": DoclingCSVParser(),
-            ".json": JSONParser(),  # Keep JSON parser (specialized handling)
-            # Text/markup formats
-            ".md": MarkdownParser(),  # Keep markdown parser (specialized handling)
-            ".mdx": MarkdownParser(),
-            ".rst": RstParser(),
-            ".adoc": DoclingAsciiDocParser(),
-            ".asciidoc": DoclingAsciiDocParser(),
-            # Images (with OCR) - only use Docling when OCR is enabled
-            ".png": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),
-            ".jpg": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),
-            ".jpeg": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),
-            ".tiff": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),
-            ".tif": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),
-            ".bmp": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),
-            ".webp": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),
-            # Media/subtitles
-            ".vtt": DoclingVTTParser(),
-            **_build_audio_parser_mapping(),
-            # Specialized XML formats
-            ".xml": DoclingXMLParser(),
-            # Formats docling doesn't support - use standard parsers
-            ".epub": EpubParser(),
-        }
     except ImportError:
-        logging.warning(
-            "docling is not installed. Using standard parsers. "
-            "For advanced document parsing, install with: pip install docling"
+        logging.log(
+            missing_log_level,
+            "docling is not installed. Using standard parsers%s. For layout-model "
+            "parsing, install with: pip install -r application/requirements-docling.txt",
+            " with native OCR" if ocr_enabled else "",
         )
-        # Fallback to standard parsers
-        legacy_pdf: BaseParser = PDFParser()
-        if pdf_text_fast_path:
-            legacy_pdf = _wrap_pdf_fast_path(legacy_pdf)
-        return {
-            ".pdf": legacy_pdf,
-            ".docx": DocxParser(),
-            ".csv": PandasCSVParser(),
-            ".xlsx": ExcelParser(),
-            ".epub": EpubParser(),
-            ".md": MarkdownParser(),
-            ".rst": RstParser(),
-            ".html": HTMLParser(),
-            ".mdx": MarkdownParser(),
-            ".json": JSONParser(),
-            ".pptx": PPTXParser(),
-            ".png": ImageParser(),
-            ".jpg": ImageParser(),
-            ".jpeg": ImageParser(),
-            **_build_audio_parser_mapping(),
-        }
+        return _legacy_file_extractor(pdf_text_fast_path, ocr_enabled=ocr_enabled)
+
+    from application.parser.file.ocr_parser import IMAGE_SUFFIXES, resolve_ocr_backend
+
+    native = None
+    if ocr_enabled and resolve_ocr_backend() == "native":
+        native = _native_ocr_parsers(text_parser=DoclingPDFParser(ocr_enabled=False))
+    if native is not None:
+        pdf_parser, image_factory = native
+        images = _image_entries(image_factory, sorted(IMAGE_SUFFIXES))
+    else:
+        pdf_parser = DoclingPDFParser(ocr_enabled=ocr_enabled)
+        # Images (with OCR) - only use Docling when OCR is enabled
+        image_factory = (
+            (lambda: DoclingImageParser(ocr_enabled=True)) if ocr_enabled else ImageParser
+        )
+        images = _image_entries(image_factory, sorted(IMAGE_SUFFIXES))
+    if pdf_text_fast_path:
+        pdf_parser = _wrap_pdf_fast_path(pdf_parser)
+    return {
+        # Documents
+        ".pdf": pdf_parser,
+        ".docx": DoclingDocxParser(),
+        ".pptx": DoclingPPTXParser(),
+        ".xlsx": DoclingXLSXParser(),
+        # Web formats
+        ".html": DoclingHTMLParser(),
+        ".xhtml": DoclingHTMLParser(),
+        # Data formats
+        ".csv": DoclingCSVParser(),
+        ".json": JSONParser(),  # Keep JSON parser (specialized handling)
+        # Text/markup formats
+        ".md": MarkdownParser(),  # Keep markdown parser (specialized handling)
+        ".mdx": MarkdownParser(),
+        ".rst": RstParser(),
+        ".adoc": DoclingAsciiDocParser(),
+        ".asciidoc": DoclingAsciiDocParser(),
+        # Images: OCR'd by the selected backend when OCR is on, else ImageParser
+        **images,
+        # Media/subtitles
+        ".vtt": DoclingVTTParser(),
+        **_build_audio_parser_mapping(),
+        # Specialized XML formats
+        ".xml": DoclingXMLParser(),
+        # Formats docling doesn't support - use standard parsers
+        ".epub": EpubParser(),
+        # Formats only anydoc reads (legacy/macro Office, OpenDocument, RTF)
+        **_gained_format_entries(),
+    }
+
+
+def _anydoc_file_extractor(ocr_enabled: bool, pdf_text_fast_path: bool = False) -> Dict[str, BaseParser]:
+    """anydoc parser map, layered over the docling (or legacy) map.
+
+    anydoc takes every format it converts; each of those parsers gets the
+    base map's parser for the same suffix as ``fallback_parser``, so a
+    scanned PDF still reaches OCR — docling's when it is installed and
+    selected, the native OCR parser otherwise — and the legacy parser when
+    OCR is off. docling, when installed, also keeps serving what anydoc
+    cannot: ``.adoc``/``.vtt``/``.xml``. Images go to whichever OCR backend
+    is active. HTML goes to the markdownify-based ``HTMLMarkdownParser``.
+
+    ``pdf_text_fast_path`` is deliberately ignored here: anydoc already reads
+    the text layer in milliseconds *and* keeps headings/tables, and does its
+    own scanned-PDF detection, so putting pypdfium2 in front of it would
+    only lose structure.
+
+    Args:
+        ocr_enabled: Enable OCR in the docling fallback/image parsers.
+        pdf_text_fast_path: Honored only when anydoc is not installed and the
+            map degrades to the base engine.
+
+    Returns:
+        Dict[str, BaseParser]: Parser keyed by lower-case file suffix.
+    """
+    from application.parser.file.anydoc_parser import (
+        ANYDOC_SUFFIXES,
+        AnydocParser,
+        anydoc_available,
+    )
+    from application.parser.file.html_parser import HTMLMarkdownParser
+
+    if not anydoc_available():
+        logging.warning(
+            "firecrawl-anydoc is not installed; DOC_PARSER_ENGINE=anydoc is using "
+            "the %s parsers instead. Install with: pip install firecrawl-anydoc",
+            "docling" if module_available("docling") else "standard",
+        )
+        return _docling_file_extractor(ocr_enabled, pdf_text_fast_path)
+
+    base = _docling_file_extractor(
+        ocr_enabled, pdf_text_fast_path=False, missing_log_level=logging.INFO
+    )
+    extractor = dict(base)
+    for suffix in ANYDOC_SUFFIXES:
+        fallback = base.get(suffix)
+        if isinstance(fallback, AnydocParser):
+            # A gained-format entry from the base map is already a
+            # fallback-less AnydocParser — anydoc delegating to anydoc would
+            # just repeat the same failure.
+            continue
+        extractor[suffix] = AnydocParser(fallback_parser=fallback)
+    extractor[".html"] = HTMLMarkdownParser()
+    extractor[".xhtml"] = HTMLMarkdownParser()
+    return extractor
+
+
+def get_default_file_extractor(
+    ocr_enabled: Optional[bool] = None,
+    pdf_text_fast_path: bool = False,
+    engine: Optional[str] = None,
+) -> Dict[str, BaseParser]:
+    """Get the default file extractor.
+
+    The engine is ``settings.DOC_PARSER_ENGINE`` unless overridden:
+
+    * ``anydoc`` (default): firecrawl-anydoc for office/PDF/tabular formats,
+      markdownify for HTML, the active OCR backend for images and as the
+      fallback for scanned PDFs; docling (if installed) for the rest anydoc
+      rejects, legacy parsers otherwise.
+    * ``docling``: the docling map, with the legacy parsers when docling is
+      not installed.
+
+    Args:
+        ocr_enabled: Enable OCR for scanned PDFs and images, through the
+            backend ``OCR_BACKEND`` resolves to. Defaults to
+            ``settings.OCR_ENABLED``.
+        pdf_text_fast_path: Read PDFs via their embedded text layer
+            (pypdfium2) instead of docling, falling back to docling for scans.
+            Off by default: it trades docling's structural markdown for speed,
+            which suits attachments (read into a prompt) but not source
+            ingestion (chunked and embedded for retrieval). A no-op under the
+            anydoc engine, which is already that fast and keeps structure.
+        engine: ``"anydoc"`` or ``"docling"``; ``None`` reads the setting.
+
+    Returns:
+        Dict[str, BaseParser]: Parser keyed by lower-case file suffix.
+    """
+    if ocr_enabled is None:
+        ocr_enabled = settings.OCR_ENABLED
+    selected = (engine or getattr(settings, "DOC_PARSER_ENGINE", None) or "anydoc")
+    selected = str(selected).strip().lower()
+    if selected == "docling":
+        return _docling_file_extractor(ocr_enabled, pdf_text_fast_path)
+    if selected != "anydoc":
+        logging.warning("Unknown DOC_PARSER_ENGINE %r; using anydoc", selected)
+    return _anydoc_file_extractor(ocr_enabled, pdf_text_fast_path)
 
 
 # For backwards compatibility
@@ -311,6 +512,15 @@ class SimpleDirectoryReader(BaseReader):
                     data = parser.parse_file(input_file, errors=self.errors)
                     parser_metadata = parser.get_file_metadata(input_file)
                 else:
+                    from application.parser.file.anydoc_parser import ANYDOC_GAINED_SUFFIXES
+
+                    if suffix_lower in ANYDOC_GAINED_SUFFIXES:
+                        # Binary office formats only anydoc reads; without it
+                        # the standard read would index OLE/zip bytes as text.
+                        raise DocumentParseError(
+                            f"No parser is available for {suffix_lower} files: "
+                            "firecrawl-anydoc is not installed (pip install firecrawl-anydoc)"
+                        )
                     # do standard read
                     with open(input_file, "r", errors=self.errors) as f:
                         data = f.read()

application/parser/file/constants.py

@@ -2,6 +2,7 @@
 
 import os
 
+from application.parser.file.anydoc_parser import ANYDOC_GAINED_SUFFIXES
 from application.stt.constants import SUPPORTED_AUDIO_EXTENSIONS
 
 
@@ -18,6 +19,12 @@ SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS = (
     ".json",
     ".xlsx",
     ".pptx",
+    # Read by the HTML parsers on every engine.
+    ".xhtml",
+    # Read by the anydoc engine (legacy/macro Office, OpenDocument, RTF).
+    # Parseable regardless of DOC_PARSER_ENGINE: anydoc is a core dependency,
+    # and both parser maps route these suffixes to it.
+    *ANYDOC_GAINED_SUFFIXES,
 )
 
 SUPPORTED_SOURCE_IMAGE_EXTENSIONS = (".png", ".jpg", ".jpeg")

application/parser/file/docling_parser.py

@@ -10,12 +10,20 @@ import importlib.util
 import logging
 import os
 import re
+import shutil
+import sys
 import tempfile
 import zipfile
 from pathlib import Path
 from typing import Dict, List, Optional, Tuple, Union
 
-from application.parser.file.base_parser import BaseParser, DocumentParseError
+from application.parser.file.base_parser import (
+    BaseParser,
+    DocumentParseError,
+    delegate_parse as _delegate,
+)
+from application.parser.file.ocr_parser import VALID_OCR_ENGINES as _VALID_OCR_ENGINES
+from application.parser.file.ocr_parser import collapse_cjk_spaces
 from application.utils import truncate_to_line_boundary
 
 logger = logging.getLogger(__name__)
@@ -67,6 +75,131 @@ def _apply_inference_settings() -> None:
         inference.compile_torch_models = settings.DOCLING_COMPILE_TORCH_MODELS
 
 
+
+
+def _resolve_ocr_engine(requested: Optional[str]) -> str:
+    """Resolve the OCR engine to build, degrading to ``auto`` when unavailable.
+
+    ``auto`` is docling's own selection (ocrmac on macOS, rapidocr on a
+    typical Linux server). The degradation is deliberate: OCR is switched on
+    by deployments that expect scans to work, so a missing engine must warn
+    and OCR with what exists rather than fail every parse.
+
+    Args:
+        requested: Engine name, or None to read ``settings.OCR_ENGINE``.
+
+    Returns:
+        One of ``_VALID_OCR_ENGINES``, guaranteed buildable here.
+    """
+    from application.core.settings import settings
+    from application.parser.file.base_parser import module_available
+
+    engine = str(requested or settings.OCR_ENGINE or "auto").strip().lower()
+    if engine not in _VALID_OCR_ENGINES:
+        logger.warning(
+            f"Unknown OCR_ENGINE {engine!r}; using docling auto-selection"
+        )
+        return "auto"
+    if engine == "tesseract" and shutil.which("tesseract") is None:
+        logger.warning(
+            "OCR_ENGINE=tesseract but no tesseract binary is on PATH (install "
+            "tesseract-ocr plus language packs: Docker builds need "
+            "INSTALL_TESSERACT=true); using docling auto-selection"
+        )
+        return "auto"
+    if engine == "ocrmac" and (
+        sys.platform != "darwin" or not module_available("ocrmac")
+    ):
+        logger.warning(
+            "OCR_ENGINE=ocrmac needs macOS with the ocrmac package; "
+            "using docling auto-selection"
+        )
+        return "auto"
+    if engine == "rapidocr" and not module_available("rapidocr"):
+        logger.warning(
+            "OCR_ENGINE=rapidocr but rapidocr is not installed; "
+            "using docling auto-selection"
+        )
+        return "auto"
+    return engine
+
+
+def _build_ocr_options(
+    engine: str, languages: Optional[List[str]], force_full_page_ocr: bool
+):
+    """docling OCR options for a resolved classic engine.
+
+    Returns None for ``auto`` (docling's default pipeline options already run
+    auto-selection) and on any build failure — the parse then proceeds on the
+    default engine rather than failing; the caller re-applies
+    ``force_full_page_ocr`` onto whatever options end up active.
+
+    Args:
+        engine: A ``_resolve_ocr_engine`` result other than ``deepseek``.
+        languages: Engine-specific language list; None uses the engine's
+            default (tesseract reads ``settings.OCR_LANGS``).
+        force_full_page_ocr: OCR whole pages instead of only bitmap regions.
+    """
+    if engine == "auto":
+        return None
+    from application.core.settings import settings
+
+    try:
+        if engine == "tesseract":
+            from docling.datamodel.pipeline_options import TesseractCliOcrOptions
+
+            from application.parser.file.ocr_parser import tesseract_languages
+
+            langs = (
+                languages
+                or [lang.strip() for lang in settings.OCR_LANGS.split("+") if lang.strip()]
+                or ["eng"]
+            )
+            # docling swallows tesseract's per-region failures: a language list
+            # with no installed pack logs an error per region and exports ''
+            # which the dropout guard then indexes as "text-sparse". Drop the
+            # packs that are not installed up front and say so.
+            installed = tesseract_languages()
+            if installed is not None:
+                missing = [lang for lang in langs if lang not in installed]
+                if missing:
+                    kept = [lang for lang in langs if lang in installed]
+                    if not kept and "eng" in installed:
+                        kept = ["eng"]
+                    logger.warning(
+                        "OCR_LANGS lists tesseract language pack(s) that are not installed: %s; "
+                        "OCR-ing with %s (install tesseract-ocr-<lang> for each missing pack)",
+                        "+".join(missing),
+                        "+".join(kept) if kept else "the requested list anyway",
+                    )
+                    langs = kept or langs
+            return TesseractCliOcrOptions(
+                lang=langs, force_full_page_ocr=force_full_page_ocr
+            )
+        if engine == "rapidocr":
+            from docling.datamodel.pipeline_options import RapidOcrOptions
+
+            return RapidOcrOptions(
+                lang=languages or ["english"],
+                force_full_page_ocr=force_full_page_ocr,
+            )
+        if engine == "ocrmac":
+            from docling.datamodel.pipeline_options import OcrMacOptions
+
+            if languages:
+                return OcrMacOptions(
+                    lang=languages, force_full_page_ocr=force_full_page_ocr
+                )
+            return OcrMacOptions(force_full_page_ocr=force_full_page_ocr)
+    except ImportError as e:
+        logger.warning(f"Failed to build {engine} OCR options: {e}")
+        return None
+    except Exception as e:
+        logger.error(f"Error building {engine} OCR options: {e}")
+        return None
+    return None
+
+
 def _tabular_content_size(file: Path) -> int:
     """Effective content size of a tabular file, in bytes.
 
@@ -118,46 +251,6 @@ def _exceeds_tabular_gate(file: Path) -> bool:
     return _tabular_content_size(file) > max_bytes
 
 
-def _delegate(
-    parser: BaseParser, file: Path, errors: str
-) -> Union[str, List[str]]:
-    """Run a lightweight fallback parser, normalizing its failures.
-
-    Anything the fallback raises is converted to ``DocumentParseError``,
-    which matters twice over. The Celery tasks that own these paths list
-    ``DocumentParseError`` in ``dont_autoretry_for``, so a deterministic
-    content error now fails once instead of retrying four times (each retry
-    re-downloading the file from S3); and ``SimpleDirectoryReader.load_data``
-    catches only ``DocumentParseError``, so one poison spreadsheet is skipped
-    into ``failed_files`` instead of aborting an entire multi-file ingest.
-
-    This restores the symmetry with ``DoclingParser.parse_file``, whose own
-    failures are already wrapped — the delegation branch bypassed that.
-
-    Args:
-        parser: The lightweight parser to delegate to.
-        file: Path to the file being parsed.
-        errors: Decoding error policy, forwarded to the parser.
-
-    Returns:
-        The parsed text, or list of row strings.
-
-    Raises:
-        DocumentParseError: If the fallback parser fails for any reason.
-    """
-    try:
-        return parser.parse_file(file, errors)
-    except DocumentParseError:
-        raise
-    except Exception as e:
-        logger.error(
-            f"Lightweight parse of {file.name} failed: {e}", exc_info=True
-        )
-        raise DocumentParseError(
-            f"Failed to parse {file.name}: the file could not be read."
-        ) from e
-
-
 def _capped_markup_copy(file: Path) -> Optional[str]:
     """Head-truncate an oversized markup file to a temp copy for docling.
 
@@ -242,7 +335,7 @@ def _ocr_min_chars_per_page() -> int:
         return int(
             getattr(
                 settings,
-                "DOCLING_OCR_MIN_CHARS_PER_PAGE",
+                "OCR_MIN_CHARS_PER_PAGE",
                 _DEFAULT_OCR_MIN_CHARS_PER_PAGE,
             )
         )
@@ -339,7 +432,7 @@ class DoclingParser(BaseParser):
     - Advanced PDF layout analysis
     - Table structure recognition
     - Reading order detection
-    - OCR for scanned documents (supports RapidOCR)
+    - OCR for scanned documents (engine chosen by ``OCR_ENGINE``)
     - Unified DoclingDocument format
     - Export to Markdown
 
@@ -353,7 +446,7 @@ class DoclingParser(BaseParser):
         ocr_enabled: bool = True,
         table_structure: bool = True,
         export_format: str = "markdown",
-        use_rapidocr: bool = True,
+        ocr_engine: Optional[str] = None,
         ocr_languages: Optional[List[str]] = None,
         force_full_page_ocr: bool = False,
     ):
@@ -363,29 +456,36 @@ class DoclingParser(BaseParser):
             ocr_enabled: Enable OCR for bitmap/image regions in documents
             table_structure: Enable table structure recognition
             export_format: Output format ('markdown', 'text', 'html')
-            use_rapidocr: Use RapidOCR engine (default True, works well in Docker)
-            ocr_languages: List of OCR languages (default: ['english'])
+            ocr_engine: OCR engine when OCR is enabled — one of
+                ``tesseract | auto | ocrmac | rapidocr | deepseek``. None
+                reads ``settings.OCR_ENGINE`` at converter build time; an
+                unavailable engine degrades to docling's auto-selection with
+                a warning.
+            ocr_languages: Engine-specific language list; None keeps the
+                engine's own default (tesseract reads ``settings.OCR_LANGS``).
             force_full_page_ocr: Force OCR on entire page (False = smart hybrid OCR)
         """
         super().__init__()
         self.ocr_enabled = ocr_enabled
         self.table_structure = table_structure
         self.export_format = export_format
-        self.use_rapidocr = use_rapidocr
-        self.ocr_languages = ocr_languages or ["english"]
+        self.ocr_engine = ocr_engine
+        self.ocr_languages = ocr_languages
         self.force_full_page_ocr = force_full_page_ocr
+        # Engine the current converter was built for (None with OCR off);
+        # drives tesseract-specific post-processing of the export.
+        self._active_ocr_engine: Optional[str] = None
         self._converter = None
 
     def _create_converter(self):
-        """Create a docling converter with hybrid OCR configuration.
+        """Create a docling converter for the configured OCR engine.
 
-        Uses smart OCR approach:
-        - When ocr_enabled=True and force_full_page_ocr=False (default):
-          Layout model detects text vs bitmap regions, OCR only runs on bitmaps
-        - When ocr_enabled=True and force_full_page_ocr=True:
-          OCR runs on entire page (for scanned documents/images)
-        - When ocr_enabled=False:
-          No OCR, only native text extraction
+        - ``ocr_enabled=False``: no OCR, native text extraction only.
+        - Classic engines (tesseract / auto / ocrmac / rapidocr): the standard
+          PDF pipeline (layout + TableFormer) with that engine's OCR options.
+          ``force_full_page_ocr=False`` (default) OCRs only the bitmap regions
+          the layout model finds; True routes whole pages through OCR.
+        - ``deepseek``: the VLM pipeline instead (``_create_vlm_converter``).
 
         Returns:
             DocumentConverter instance
@@ -400,6 +500,11 @@ class DoclingParser(BaseParser):
 
         _apply_inference_settings()
 
+        engine = _resolve_ocr_engine(self.ocr_engine) if self.ocr_enabled else None
+        self._active_ocr_engine = engine
+        if engine == "deepseek":
+            return self._create_vlm_converter()
+
         pipeline_options = PdfPipelineOptions(
             do_ocr=self.ocr_enabled,
             do_table_structure=self.table_structure,
@@ -407,13 +512,15 @@ class DoclingParser(BaseParser):
         _apply_pipeline_caps(pipeline_options)
 
         if self.ocr_enabled:
-            ocr_options = self._get_ocr_options()
+            ocr_options = _build_ocr_options(
+                engine, self.ocr_languages, self.force_full_page_ocr
+            )
             if ocr_options is not None:
                 pipeline_options.ocr_options = ocr_options
             # Docling's *default* OCR options carry their own flag, so without
-            # this the setting was silently dropped whenever `_get_ocr_options`
-            # returned None (use_rapidocr=False) — including the dropout retry,
-            # whose whole point is forcing full-page OCR.
+            # this the setting was silently dropped whenever no explicit
+            # options were built (engine=auto, or a build failure) — including
+            # the dropout retry, whose whole point is forcing full-page OCR.
             active_ocr_options = getattr(pipeline_options, "ocr_options", None)
             if hasattr(active_ocr_options, "force_full_page_ocr"):
                 active_ocr_options.force_full_page_ocr = self.force_full_page_ocr
@@ -429,17 +536,64 @@ class DoclingParser(BaseParser):
             }
         )
 
+    def _create_vlm_converter(self):
+        """DeepSeek-OCR converter via docling's VLM pipeline.
+
+        Each page goes to an OpenAI-compatible endpoint (Ollama or vLLM;
+        ``OCR_DEEPSEEK_URL`` / ``OCR_DEEPSEEK_MODEL``) and the grounded output
+        is parsed back into a DoclingDocument. This replaces the *entire*
+        classic pipeline — no layout/TableFormer/OCR models load in the worker
+        (~370 MB RSS vs 1.0-1.6 GB measured), the compute lives in the model
+        server. Bench trade-offs (2026-08): best table/CJK/degraded-scan
+        quality of every engine tried, ~10-20 s/page on modest hardware, and
+        occasional silent drops of page-level elements (titles).
+        """
+        from docling.datamodel import vlm_model_specs
+        from docling.datamodel.pipeline_options import VlmPipelineOptions
+        from docling.document_converter import (
+            DocumentConverter,
+            ImageFormatOption,
+            InputFormat,
+            PdfFormatOption,
+        )
+        from docling.pipeline.vlm_pipeline import VlmPipeline
+
+        from application.core.settings import settings
+
+        vlm_options = vlm_model_specs.DEEPSEEKOCR_OLLAMA.model_copy(deep=True)
+        vlm_options.url = settings.OCR_DEEPSEEK_URL
+        vlm_options.params["model"] = settings.OCR_DEEPSEEK_MODEL
+        # docling's spec default is 90 s per request, which a laptop-hosted
+        # 3B model overruns; honour the same per-page budget the native
+        # backend gives the endpoint.
+        vlm_options.timeout = float(settings.OCR_DEEPSEEK_TIMEOUT)
+        pipeline_options = VlmPipelineOptions(
+            vlm_options=vlm_options, enable_remote_services=True
+        )
+        return DocumentConverter(
+            format_options={
+                InputFormat.PDF: PdfFormatOption(
+                    pipeline_cls=VlmPipeline, pipeline_options=pipeline_options
+                ),
+                InputFormat.IMAGE: ImageFormatOption(
+                    pipeline_cls=VlmPipeline, pipeline_options=pipeline_options
+                ),
+            }
+        )
+
     def _init_parser(self) -> Dict:
         """Initialize the docling converter with hybrid OCR."""
+        from application.core.settings import settings
+
         logger.info("Initializing DoclingParser...")
         logger.info(f"  ocr_enabled={self.ocr_enabled}")
         logger.info(f"  force_full_page_ocr={self.force_full_page_ocr}")
-        logger.info(f"  use_rapidocr={self.use_rapidocr}")
+        logger.info(f"  ocr_engine={self.ocr_engine or settings.OCR_ENGINE}")
 
         if importlib.util.find_spec("docling.document_converter") is None:
             raise ImportError(
                 "docling is required for DoclingParser. "
-                "Install it with: pip install docling"
+                "Install it with: pip install -r application/requirements-docling.txt"
             )
 
         # Create converter with hybrid OCR (smart: text direct, bitmaps OCR'd)
@@ -450,34 +604,11 @@ class DoclingParser(BaseParser):
             "ocr_enabled": self.ocr_enabled,
             "table_structure": self.table_structure,
             "export_format": self.export_format,
-            "use_rapidocr": self.use_rapidocr,
+            "ocr_engine": self.ocr_engine,
             "ocr_languages": self.ocr_languages,
             "force_full_page_ocr": self.force_full_page_ocr,
         }
 
-    def _get_ocr_options(self):
-        """Get OCR options based on configuration.
-
-        Returns RapidOcrOptions if use_rapidocr is True and available,
-        otherwise returns None to use docling defaults.
-        """
-        if not self.use_rapidocr:
-            return None
-
-        try:
-            from docling.datamodel.pipeline_options import RapidOcrOptions
-
-            return RapidOcrOptions(
-                lang=self.ocr_languages,
-                force_full_page_ocr=self.force_full_page_ocr,
-            )
-        except ImportError as e:
-            logger.warning(f"Failed to import RapidOcrOptions: {e}")
-            return None
-        except Exception as e:
-            logger.error(f"Error creating RapidOcrOptions: {e}")
-            return None
-
     def _export_content(self, document) -> str:
         """Export document content in the configured format.
 
@@ -505,8 +636,25 @@ class DoclingParser(BaseParser):
                     f"Standard export minimal ({len(stripped_content)} chars), "
                     f"extracting {len(extracted_texts)} texts directly"
                 )
-                return "\n\n".join(extracted_texts)
+                content = "\n\n".join(extracted_texts)
 
+        return self._postprocess_ocr_text(content)
+
+    def _postprocess_ocr_text(self, content: str) -> str:
+        """Engine-specific cleanup of OCR'd text.
+
+        tesseract's CJK models space every glyph ("互相 保密 协议"); the native
+        backend collapses that in ``TesseractEngine`` and the docling path
+        needs the same treatment or Chinese scans index worse here than there.
+        Other engines (and OCR off) return the export untouched. Only the PDF
+        and image parsers construct with OCR on — the office/markup parsers
+        never OCR, so their born-digital text must not be rewritten. On a
+        hybrid-OCR PDF the collapse still covers the whole export, native
+        text included; deliberate spaces inside CJK runs are rare enough
+        there that indexing scans correctly wins.
+        """
+        if self.ocr_enabled and self._active_ocr_engine == "tesseract":
+            return collapse_cjk_spaces(content)
         return content
 
     def _ocr_guard_applies(self, file: Path) -> bool:
@@ -687,7 +835,7 @@ class DoclingParser(BaseParser):
 
 
 class DoclingPDFParser(DoclingParser):
-    """Docling-based PDF parser with advanced features and RapidOCR support.
+    """Docling-based PDF parser with advanced features and configurable OCR.
 
     Uses hybrid OCR approach by default:
     - Text regions: Direct PDF text extraction (fast)
@@ -700,7 +848,7 @@ class DoclingPDFParser(DoclingParser):
         self,
         ocr_enabled: bool = True,
         table_structure: bool = True,
-        use_rapidocr: bool = True,
+        ocr_engine: Optional[str] = None,
         ocr_languages: Optional[List[str]] = None,
         force_full_page_ocr: bool = False,
     ):
@@ -708,31 +856,60 @@ class DoclingPDFParser(DoclingParser):
             ocr_enabled=ocr_enabled,
             table_structure=table_structure,
             export_format="markdown",
-            use_rapidocr=use_rapidocr,
+            ocr_engine=ocr_engine,
             ocr_languages=ocr_languages,
             force_full_page_ocr=force_full_page_ocr,
         )
 
 
+    def ocr_pages(self, file: Path, indices: List[int]) -> Dict[int, str]:
+        """Convert only the given 0-based pages of ``file`` and return their text.
+
+        The counterpart of ``NativeOcrPdfParser.ocr_pages`` for the docling
+        backend: ``AnydocParser`` calls it for the scanned pages of a mixed
+        document. Each page is a separate ``convert`` with ``page_range`` so
+        the layout model sees one page at a time; a fully scanned page is one
+        bitmap region, which hybrid OCR reads in full.
+
+        Raises:
+            DocumentParseError: docling failed on a page.
+        """
+        if self._converter is None:
+            self._init_parser()
+        path = Path(file)
+        texts: Dict[int, str] = {}
+        for index in indices:
+            if index < 0:
+                continue
+            try:
+                result = self._converter.convert(str(path), page_range=(index + 1, index + 1))
+                texts[index] = self._export_content(result.document).strip()
+            except Exception as exc:
+                raise DocumentParseError(
+                    f"Failed to OCR page {index + 1} of {path.name} with docling: {exc}"
+                ) from exc
+        return texts
+
+
 class DoclingDocxParser(DoclingParser):
     """Docling-based DOCX parser."""
 
     def __init__(self):
-        super().__init__(export_format="markdown")
+        super().__init__(ocr_enabled=False, export_format="markdown")
 
 
 class DoclingPPTXParser(DoclingParser):
     """Docling-based PPTX parser."""
 
     def __init__(self):
-        super().__init__(export_format="markdown")
+        super().__init__(ocr_enabled=False, export_format="markdown")
 
 
 class DoclingXLSXParser(DoclingParser):
     """Docling-based XLSX parser with table structure."""
 
     def __init__(self):
-        super().__init__(table_structure=True, export_format="markdown")
+        super().__init__(ocr_enabled=False, table_structure=True, export_format="markdown")
 
     def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
         """Parse an XLSX file, delegating oversized files to ``ExcelParser``."""
@@ -751,7 +928,7 @@ class DoclingHTMLParser(DoclingParser):
     """Docling-based HTML parser."""
 
     def __init__(self):
-        super().__init__(export_format="markdown")
+        super().__init__(ocr_enabled=False, export_format="markdown")
 
     def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
         """Parse HTML, truncating oversized element-dense markup first."""
@@ -759,7 +936,7 @@ class DoclingHTMLParser(DoclingParser):
 
 
 class DoclingImageParser(DoclingParser):
-    """Docling-based image parser with OCR and RapidOCR support.
+    """Docling-based image parser with configurable OCR.
 
     For images, force_full_page_ocr=True is used since images are entirely
     visual and require full OCR to extract any text.
@@ -768,14 +945,14 @@ class DoclingImageParser(DoclingParser):
     def __init__(
         self,
         ocr_enabled: bool = True,
-        use_rapidocr: bool = True,
+        ocr_engine: Optional[str] = None,
         ocr_languages: Optional[List[str]] = None,
         force_full_page_ocr: bool = True,
     ):
         super().__init__(
             ocr_enabled=ocr_enabled,
             export_format="markdown",
-            use_rapidocr=use_rapidocr,
+            ocr_engine=ocr_engine,
             ocr_languages=ocr_languages,
             force_full_page_ocr=force_full_page_ocr,
         )
@@ -785,7 +962,7 @@ class DoclingCSVParser(DoclingParser):
     """Docling-based CSV parser."""
 
     def __init__(self):
-        super().__init__(table_structure=True, export_format="markdown")
+        super().__init__(ocr_enabled=False, table_structure=True, export_format="markdown")
 
     def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
         """Parse a CSV file, delegating oversized files to the plain ``CSVParser``."""
@@ -804,21 +981,21 @@ class DoclingMarkdownParser(DoclingParser):
     """Docling-based Markdown parser."""
 
     def __init__(self):
-        super().__init__(export_format="markdown")
+        super().__init__(ocr_enabled=False, export_format="markdown")
 
 
 class DoclingAsciiDocParser(DoclingParser):
     """Docling-based AsciiDoc parser."""
 
     def __init__(self):
-        super().__init__(export_format="markdown")
+        super().__init__(ocr_enabled=False, export_format="markdown")
 
 
 class DoclingVTTParser(DoclingParser):
     """Docling-based WebVTT (video text tracks) parser."""
 
     def __init__(self):
-        super().__init__(export_format="markdown")
+        super().__init__(ocr_enabled=False, export_format="markdown")
 
     def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
         """Parse WebVTT, truncating oversized cue-dense tracks first."""
@@ -829,4 +1006,4 @@ class DoclingXMLParser(DoclingParser):
     """Docling-based XML parser (USPTO, JATS)."""
 
     def __init__(self):
-        super().__init__(export_format="markdown")
+        super().__init__(ocr_enabled=False, export_format="markdown")

application/parser/file/html_parser.py

@@ -1,12 +1,160 @@
-"""HTML parser.
-
-Contains parser for html files.
+"""HTML parsers.
 
+``HTMLParser`` returns the visible text of a page (the ``fast`` engine and
+the legacy no-docling map rely on that plain-text contract).
+``HTMLMarkdownParser`` returns Markdown instead and is what the anydoc
+engine maps ``.html``/``.xhtml`` to: anydoc has no HTML support, and
+markdownify — already a dependency of the web crawler — keeps links, GFM
+tables, headings and byte-exact code blocks at a few ms per page.
 """
+import logging
+import re
 from pathlib import Path
-from typing import Dict, Union
+from typing import Dict, Optional, Tuple, Union
 
+from application.core.settings import settings
 from application.parser.file.base_parser import BaseParser
+from application.utils import truncate_to_line_boundary
+
+logger = logging.getLogger(__name__)
+
+# The crawler's conventions (``crawler_markdown.py`` / ``read_webpage.py``),
+# so file and web ingestion produce the same Markdown shape.
+MARKDOWNIFY_OPTIONS = {"heading_style": "ATX", "newline_style": "BACKSLASH"}
+
+# Elements whose text is never document content. ``title`` is reported via
+# ``get_file_metadata`` instead of leaking in as a stray first line.
+_DROP_TAGS = ("title", "script", "style", "noscript", "template")
+# Attributes whose ``data:`` URIs would otherwise land in the Markdown as
+# link/image targets: one inline image is megabytes of base64 "text" for the
+# chunker and embedder (a 5 MB data-URI ``<img>`` measured 7 MB of output).
+_URI_ATTRIBUTES = ("src", "href", "srcset", "poster", "data")
+
+
+def html_to_markdown(html: Union[str, bytes]) -> str:
+    """Convert an HTML/XHTML document to Markdown.
+
+    Args:
+        html: The markup. Pass the raw bytes when you have them: BeautifulSoup
+            then decodes by the document's own BOM / ``<meta charset>``
+            instead of the process locale, so a windows-1250 page keeps its
+            diacritics.
+
+    Returns:
+        Markdown with runs of blank lines collapsed to one.
+    """
+    from bs4 import BeautifulSoup
+
+    return soup_to_markdown(BeautifulSoup(html, "html.parser"))
+
+
+def soup_to_markdown(soup) -> str:
+    """Convert a parsed ``BeautifulSoup`` tree to Markdown.
+
+    Destructive: the non-content elements in ``_DROP_TAGS`` (including
+    ``<title>``) are removed from ``soup``; read the title first.
+
+    Args:
+        soup: The parsed document.
+
+    Returns:
+        Markdown with runs of blank lines collapsed to one.
+    """
+    from bs4 import CData, Declaration, ProcessingInstruction
+    from markdownify import MarkdownConverter
+
+    for tag in soup.find_all(_DROP_TAGS):
+        tag.decompose()
+    # ``<?xml ...?>`` (every XHTML file), CDATA and stray declarations are
+    # not text; markdownify would emit them as the document's first line.
+    for node in soup.find_all(string=lambda s: isinstance(s, (ProcessingInstruction, CData, Declaration))):
+        node.extract()
+    for tag in soup.find_all(True):
+        for attribute in _URI_ATTRIBUTES:
+            value = tag.get(attribute)
+            if isinstance(value, list):
+                value = " ".join(value)
+            if isinstance(value, str) and "data:" in value.lower():
+                del tag[attribute]
+    markdown = MarkdownConverter(**MARKDOWNIFY_OPTIONS).convert_soup(soup)
+    return re.sub(r"\n{3,}", "\n\n", markdown).strip()
+
+
+def _soup_title(soup) -> Optional[str]:
+    """The document's ``<title>`` text, or None."""
+    if soup.title and soup.title.string:
+        return str(soup.title.string)
+    return None
+
+
+def _trim_torn_utf8_tail(data: bytes) -> bytes:
+    """Drop a trailing partial multi-byte UTF-8 sequence left by a byte cut.
+
+    A cut that lands inside a multi-byte character matters more than one
+    lost glyph: BeautifulSoup decodes strictly and, when the declared UTF-8
+    fails on the torn tail, retries as windows-1252 — and if every other
+    non-ASCII byte happens to be cp1252-valid (German umlauts are), the
+    *whole* document comes back as mojibake. Only a UTF-8 tail is
+    recognised; a single-byte page loses at most the final byte.
+
+    Args:
+        data: A head window whose last bytes may be a torn sequence.
+
+    Returns:
+        ``data`` minus any trailing incomplete UTF-8 sequence.
+    """
+    end = len(data)
+    trailing = 0
+    while end > 0 and trailing < 3 and 0x80 <= data[end - 1] < 0xC0:
+        end -= 1
+        trailing += 1
+    if end == 0:
+        return data
+    lead = data[end - 1]
+    if lead < 0xC0:
+        return data
+    need = 2 if lead < 0xE0 else 3 if lead < 0xF0 else 4
+    if trailing + 1 < need:
+        return data[: end - 1]
+    return data
+
+
+def read_markup_head(file: Path, max_bytes: int) -> bytes:
+    """Read a markup file, head-truncated to ``max_bytes`` on a line boundary.
+
+    HTML has no lightweight fallback parser to bound it by other means (the
+    way oversized CSV/XLSX go to the plain tabular parsers), and the soup +
+    markdownify tree costs ~50x the input — 30 MB measured at 1.6 GB RSS,
+    against a 100 MB upload cap. Parsing only the head keeps one upload from
+    taking the ingest worker down; the cut lands on a line boundary when one
+    is reasonably close, and the lenient HTML parser copes with the torn tail.
+
+    Args:
+        file: Path to the markup file.
+        max_bytes: Size gate; ``<= 0`` reads the whole file.
+
+    Returns:
+        The file's bytes, or its first ``max_bytes`` when it is larger.
+    """
+    with open(file, "rb") as fh:
+        if max_bytes <= 0:
+            return fh.read()
+        head = fh.read(max_bytes + 1)
+    if len(head) <= max_bytes:
+        return head
+    logger.warning(
+        f"Markup {Path(file).name} exceeds MARKUP_MAX_BYTES ({max_bytes}); "
+        f"parsing the first {max_bytes} bytes to bound memory"
+    )
+    head = head[:max_bytes]
+    if head[:2] in (b"\xff\xfe", b"\xfe\xff"):
+        # UTF-16 (BOM-declared): a byte-level line cut lands between the two
+        # bytes of a code unit and the strict decode then fails outright, so
+        # keep an even byte count and let the lenient parser take the torn
+        # tail. (Only 2-byte units matter here; a torn surrogate pair is one
+        # lost character.)
+        return head[: len(head) - (len(head) % 2)]
+    return _trim_torn_utf8_tail(truncate_to_line_boundary(head))
 
 
 class HTMLParser(BaseParser):
@@ -40,3 +188,51 @@ class HTMLParser(BaseParser):
         if soup.title and soup.title.string:
             return {"title": str(soup.title.string)}
         return {}
+
+
+class HTMLMarkdownParser(HTMLParser):
+    """HTML/XHTML to Markdown, for chunking and retrieval.
+
+    Differs from ``HTMLParser`` in three ways: the body comes back as
+    Markdown; the file is read as bytes so the document's own charset
+    declaration wins over the process locale; and input is head-truncated at
+    ``MARKUP_MAX_BYTES`` (the docling HTML path has the same gate under
+    ``DOCLING_MARKUP_MAX_BYTES``).
+    """
+
+    def __init__(self, parser_config: Optional[Dict] = None) -> None:
+        super().__init__(parser_config)
+        # (path, title) of the most recent ``parse_file``, so the metadata
+        # call that follows it does not build the soup a second time.
+        self._last_title: Optional[Tuple[Path, Optional[str]]] = None
+
+    @staticmethod
+    def _read(file: Path) -> bytes:
+        return read_markup_head(Path(file), int(settings.MARKUP_MAX_BYTES))
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, list[str]]:
+        """Convert an HTML file to Markdown.
+
+        ``errors`` is accepted for the ``BaseParser`` contract but unused:
+        decoding is done by BeautifulSoup from the raw bytes.
+        """
+        from bs4 import BeautifulSoup
+
+        _ = errors
+        soup = BeautifulSoup(self._read(file), "html.parser")
+        self._last_title = (Path(file), _soup_title(soup))
+        return soup_to_markdown(soup)
+
+    def get_file_metadata(self, file: Path) -> Dict:
+        """Return the document title, decoded the same way as the body."""
+        from bs4 import BeautifulSoup
+
+        last = self._last_title
+        if last is not None and last[0] == Path(file):
+            title = last[1]
+        else:
+            try:
+                title = _soup_title(BeautifulSoup(self._read(file), "html.parser"))
+            except OSError:
+                return {}
+        return {"title": title} if title else {}

application/parser/file/ocr_parser.py

@@ -0,0 +1,872 @@
+"""Native OCR parsers: scanned PDFs and images without docling.
+
+OCR in DocsGPT has two backends, selected by ``OCR_BACKEND``:
+
+* ``docling`` — the layout-model pipeline in ``docling_parser.py``. Hybrid
+  OCR (only the bitmap regions of a page), reading-order and table
+  structure recovery, and five engines (tesseract / auto / ocrmac /
+  rapidocr / deepseek). Costs the optional docling install (torch, ONNX
+  models, gigabytes of image) and seconds to minutes per file.
+* ``native`` (this module) — page rendering with pypdfium2 and Pillow, both
+  core dependencies, feeding one of two engines directly: the system
+  ``tesseract`` binary, or a DeepSeek-OCR model behind an OpenAI-compatible
+  endpoint (Ollama / vLLM). No ML models load in the worker. Pages that
+  carry a text layer are read through pypdfium2 and never OCR'd; pages
+  without one are rendered and OCR'd. What it lacks against docling is the
+  layout model: multi-column scans rely on tesseract's own page
+  segmentation, and tesseract yields tables as plain lines (DeepSeek-OCR
+  emits Markdown tables itself).
+
+``auto`` picks docling when it is installed and native otherwise, so a
+deployment that never installs the docling extra gets working OCR from the
+tesseract binary alone, and one that does install it keeps today's
+behaviour unchanged.
+"""
+import base64
+import functools
+import io
+import logging
+import re
+import shutil
+import subprocess
+from pathlib import Path
+from typing import Dict, List, Optional, Protocol, Tuple, Union
+
+from application.parser.file.base_parser import (
+    BaseParser,
+    DocumentParseError,
+    delegate_parse,
+    module_available,
+)
+
+logger = logging.getLogger(__name__)
+
+# Every engine ``OCR_ENGINE`` accepts. ``auto``, ``ocrmac`` and ``rapidocr``
+# exist only inside docling; the native backend maps them to tesseract.
+VALID_OCR_ENGINES: Tuple[str, ...] = ("tesseract", "auto", "ocrmac", "rapidocr", "deepseek")
+NATIVE_OCR_ENGINES: Tuple[str, ...] = ("tesseract", "deepseek")
+VALID_OCR_BACKENDS: Tuple[str, ...] = ("auto", "docling", "native")
+
+IMAGE_SUFFIXES = frozenset({".png", ".jpg", ".jpeg", ".tiff", ".tif", ".bmp", ".webp"})
+
+# Characters in a page's text layer at or above which the page is read from
+# that layer instead of being OCR'd. Same separation ``PdfiumTextParser``
+# relies on: scanned pages sit at 0-17 chars, text pages in the hundreds+.
+DEFAULT_TEXT_LAYER_MIN_CHARS = 32
+# Chars-per-page floor for the near-empty guard when the setting is unusable.
+_DEFAULT_MIN_CHARS_PER_PAGE = 20
+_DEFAULT_RENDER_DPI = 200
+_MIN_RENDER_DPI, _MAX_RENDER_DPI = 72, 600
+# Pixel budget for one rendered page. pypdfium2 allocates width*height*4
+# bytes up front with no cap of its own, so a 14400x14400 pt page (the PDF
+# maximum; a 191-byte file) at 200 dpi would ask for 40000x40000 px = 6 GiB.
+# 40 MP is ~10x a Letter page at 200 dpi and still OCRs an A0 poster at a
+# usable ~125 dpi; anything larger renders at the scale that fits.
+_MAX_RENDER_PIXELS = 40_000_000
+_TESSERACT_TIMEOUT_SECONDS = 300
+# tesseract reports a missing language pack on stderr and, when at least one
+# other requested pack loads, exits 0 and silently OCRs with what it has —
+# so the exit code alone cannot catch OCR_LANGS=eng+chi_sim without chi_sim.
+_TESSERACT_LANG_ERROR_RE = re.compile(r"Error opening data file|Failed loading language")
+_TESSERACT_LANG_RE = re.compile(r"^[A-Za-z0-9_/\-]+$")
+# docling's DeepSeek-OCR prompt minus its ``<|grounding|>`` prefix: grounding
+# makes the model wrap every element in ref/det tags with bounding boxes,
+# which docling parses back into a layout tree. Plain Markdown is what the
+# ingestion pipeline stores, so ask for that directly.
+DEEPSEEK_PROMPT = "Convert the document to markdown."
+_DEEPSEEK_MAX_TOKENS = 4096
+# Defensive cleanup should a served model still emit grounding markup.
+_DEEPSEEK_DET_BLOCK_RE = re.compile(r"<\|det\|>.*?<\|/det\|>", re.DOTALL)
+_DEEPSEEK_TAG_RE = re.compile(r"<\|/?ref\|>|<\|grounding\|>|<\|end▁of▁sentence\|>")
+# tesseract's CJK models emit one space between every character ("互相 保密 协议")
+# because they segment by glyph. Chinese and Japanese are written without
+# word spaces, so collapse whitespace between two CJK characters (or CJK
+# punctuation); spaces next to Latin text, digits and line breaks stay.
+_CJK_CHAR = r"[\u3000-\u303f\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uff00-\uffef]"
+_CJK_SPACE_RE = re.compile(rf"(?<={_CJK_CHAR})[ \t]+(?={_CJK_CHAR})")
+
+
+class OcrUnavailableError(DocumentParseError):
+    """The selected OCR engine cannot run on this host (missing binary or endpoint)."""
+
+
+class OcrEngine(Protocol):
+    """Anything that turns one page image into text."""
+
+    name: str
+
+    def ocr_image(self, image) -> str:  # pragma: no cover - protocol
+        """Return the text (or Markdown) recognised in ``image`` (a PIL image)."""
+
+
+# ---------------------------------------------------------------------------
+# Resolution
+# ---------------------------------------------------------------------------
+
+
+def native_ocr_available() -> bool:
+    """Whether the native backend's rendering stack (pypdfium2 + Pillow) is importable."""
+    return module_available("pypdfium2") and module_available("PIL")
+
+
+def resolve_ocr_backend(requested: Optional[str] = None) -> str:
+    """Resolve ``OCR_BACKEND`` to the backend that will actually run.
+
+    Args:
+        requested: ``auto`` | ``docling`` | ``native``, or None to read the setting.
+
+    Returns:
+        ``"docling"`` or ``"native"``. ``auto`` prefers docling when it is
+        installed; ``docling`` without the install degrades to native with a
+        warning rather than leaving OCR off.
+    """
+    from application.core.settings import settings
+
+    backend = str(requested or getattr(settings, "OCR_BACKEND", None) or "auto").strip().lower()
+    if backend not in VALID_OCR_BACKENDS:
+        logger.warning(f"Unknown OCR_BACKEND {backend!r}; using auto")
+        backend = "auto"
+    docling_installed = module_available("docling")
+    if backend == "docling" and not docling_installed:
+        logger.warning(
+            "OCR_BACKEND=docling but docling is not installed (pip install -r "
+            "application/requirements-docling.txt); using the native OCR backend"
+        )
+        return "native"
+    if backend == "auto":
+        return "docling" if docling_installed else "native"
+    return backend
+
+
+def resolve_native_ocr_engine(requested: Optional[str] = None) -> str:
+    """Map ``OCR_ENGINE`` onto an engine the native backend implements.
+
+    Args:
+        requested: Engine name, or None to read ``settings.OCR_ENGINE``.
+
+    Returns:
+        ``"tesseract"`` or ``"deepseek"``. docling-only engines (``auto``,
+        ``ocrmac``, ``rapidocr``) and unknown names become tesseract with a
+        warning, so switching a deployment off docling never silently
+        disables OCR.
+    """
+    from application.core.settings import settings
+
+    engine = str(requested or getattr(settings, "OCR_ENGINE", None) or "tesseract").strip().lower()
+    if engine in NATIVE_OCR_ENGINES:
+        return engine
+    if engine in VALID_OCR_ENGINES:
+        logger.warning(
+            f"OCR_ENGINE={engine!r} is only available through the docling backend; "
+            "the native backend is using tesseract"
+        )
+    else:
+        logger.warning(f"Unknown OCR_ENGINE {engine!r}; the native backend is using tesseract")
+    return "tesseract"
+
+
+def ocr_min_chars_per_page() -> int:
+    """Chars-per-page floor for the near-empty OCR guard (``OCR_MIN_CHARS_PER_PAGE``); 0 disables it."""
+    from application.core.settings import settings
+
+    try:
+        return int(getattr(settings, "OCR_MIN_CHARS_PER_PAGE", _DEFAULT_MIN_CHARS_PER_PAGE))
+    except (TypeError, ValueError):
+        return _DEFAULT_MIN_CHARS_PER_PAGE
+
+
+def render_dpi() -> int:
+    """Rendering resolution for pages that need OCR (``OCR_RENDER_DPI``), clamped to a sane range."""
+    from application.core.settings import settings
+
+    try:
+        dpi = int(getattr(settings, "OCR_RENDER_DPI", _DEFAULT_RENDER_DPI))
+    except (TypeError, ValueError):
+        dpi = _DEFAULT_RENDER_DPI
+    return max(_MIN_RENDER_DPI, min(_MAX_RENDER_DPI, dpi))
+
+
+def _has_alpha(image) -> bool:
+    return "A" in image.getbands() or (image.mode == "P" and "transparency" in image.info)
+
+
+def _png_bytes(image) -> bytes:
+    """Encode a PIL image as PNG, flattening modes the engines cannot take.
+
+    Transparency is composited onto white: a plain ``convert("RGB")`` drops
+    the alpha channel and leaves transparent pixels black, which turns dark
+    text on a transparent background into an all-black page that OCRs to
+    nothing.
+    """
+    from PIL import Image
+
+    if _has_alpha(image):
+        rgba = image.convert("RGBA")
+        canvas = Image.new("RGB", rgba.size, "white")
+        canvas.paste(rgba, mask=rgba.getchannel("A"))
+        image = canvas
+    elif image.mode not in ("RGB", "L"):
+        image = image.convert("RGB")
+    buffer = io.BytesIO()
+    image.save(buffer, format="PNG")
+    return buffer.getvalue()
+
+
+def fit_to_pixel_budget(image):
+    """Downscale a decoded image to ``_MAX_RENDER_PIXELS`` before OCR.
+
+    The same budget the PDF renderer applies: a 28 KB PNG can decode to
+    81 MP (Pillow's bomb guard only trips at ~178 MP), and every extra
+    pixel is paid again in the PNG re-encode and inside tesseract.
+    """
+    from PIL import Image
+
+    width, height = image.size
+    scale = render_scale(width, height, 72)
+    if scale >= 1.0:
+        return image
+    if image.mode not in ("RGB", "L", "RGBA", "LA"):
+        image = image.convert("RGBA" if _has_alpha(image) else "RGB")
+    logger.warning(
+        "Image is %dx%d px; downscaling to %.0f%% to stay within %d MP before OCR",
+        width,
+        height,
+        scale * 100,
+        _MAX_RENDER_PIXELS // 1_000_000,
+    )
+    return image.resize(
+        (max(1, int(width * scale)), max(1, int(height * scale))), Image.Resampling.BILINEAR
+    )
+
+
+@functools.lru_cache(maxsize=1)
+def tesseract_languages() -> Optional[frozenset]:
+    """Language packs the ``tesseract`` binary reports (``--list-langs``), or None when unknown.
+
+    Cached for the process: packs are installed with the image, not at runtime.
+    """
+    if shutil.which("tesseract") is None:
+        return None
+    try:
+        completed = subprocess.run(
+            ["tesseract", "--list-langs"], capture_output=True, timeout=30, check=False
+        )
+    except (OSError, subprocess.TimeoutExpired):
+        return None
+    output = completed.stdout.decode("utf-8", "replace") + "\n" + completed.stderr.decode("utf-8", "replace")
+    langs = frozenset(
+        line.strip() for line in output.splitlines() if _TESSERACT_LANG_RE.match(line.strip())
+    )
+    return langs or None
+
+
+# ---------------------------------------------------------------------------
+# Engines
+# ---------------------------------------------------------------------------
+
+
+class TesseractEngine:
+    """OCR through the system ``tesseract`` binary (stdin -> stdout, no Python wrapper).
+
+    Attributes:
+        languages: Tesseract language codes; None reads ``OCR_LANGS`` per call.
+        psm: Page segmentation mode (3 = fully automatic, tesseract's default).
+        timeout: Seconds allowed per page.
+    """
+
+    name = "tesseract"
+
+    def __init__(
+        self,
+        languages: Optional[List[str]] = None,
+        psm: int = 3,
+        timeout: float = _TESSERACT_TIMEOUT_SECONDS,
+    ) -> None:
+        self.languages = languages
+        self.psm = psm
+        self.timeout = timeout
+
+    @staticmethod
+    def available() -> bool:
+        """Whether a ``tesseract`` binary is on PATH."""
+        return shutil.which("tesseract") is not None
+
+    def _language_arg(self) -> str:
+        if self.languages:
+            langs = [lang.strip() for lang in self.languages if lang and lang.strip()]
+        else:
+            from application.core.settings import settings
+
+            configured = str(getattr(settings, "OCR_LANGS", "") or "eng")
+            langs = [lang.strip() for lang in configured.split("+") if lang.strip()]
+        return "+".join(langs) or "eng"
+
+    def command(self) -> List[str]:
+        """The tesseract command line, for logging and tests."""
+        return ["tesseract", "stdin", "stdout", "-l", self._language_arg(), "--psm", str(self.psm)]
+
+    def ocr_image(self, image) -> str:
+        """OCR one PIL image.
+
+        Raises:
+            OcrUnavailableError: No tesseract binary on PATH.
+            DocumentParseError: tesseract failed or timed out.
+        """
+        if not self.available():
+            raise OcrUnavailableError(
+                "OCR_ENGINE=tesseract but no tesseract binary is on PATH. Install "
+                "tesseract-ocr plus language packs (Docker: build with "
+                "INSTALL_TESSERACT=true; local: apt/brew install tesseract-ocr), "
+                "or set OCR_ENGINE=deepseek / install the docling extra."
+            )
+        try:
+            completed = subprocess.run(
+                self.command(),
+                input=_png_bytes(image),
+                capture_output=True,
+                timeout=self.timeout,
+                check=False,
+            )
+        except subprocess.TimeoutExpired as exc:
+            raise DocumentParseError(f"tesseract timed out after {self.timeout:.0f}s on a page") from exc
+        except OSError as exc:
+            raise DocumentParseError(f"tesseract could not be started: {exc}") from exc
+        stderr = completed.stderr.decode("utf-8", "replace").strip()
+        if _TESSERACT_LANG_ERROR_RE.search(stderr):
+            # Exit code 0 here means tesseract dropped the missing pack and
+            # OCR'd with the rest: Chinese scans would come out as garbage
+            # with no signal. Fail the file and name the fix instead.
+            raise OcrUnavailableError(
+                f"tesseract could not load a language pack for OCR_LANGS={self._language_arg()!r}: "
+                f"{stderr[:300]}. Install the tesseract-ocr-<lang> package for every language listed."
+            )
+        if completed.returncode != 0:
+            raise DocumentParseError(f"tesseract failed (exit {completed.returncode}): {stderr[:300]}")
+        return collapse_cjk_spaces(completed.stdout.decode("utf-8", "replace")).strip()
+
+
+def collapse_cjk_spaces(text: str) -> str:
+    """Remove the per-glyph spaces tesseract inserts inside CJK runs."""
+    return _CJK_SPACE_RE.sub("", text)
+
+
+def clean_deepseek_output(text: str) -> str:
+    """Strip DeepSeek-OCR grounding markup, keeping the referenced text."""
+    text = _DEEPSEEK_DET_BLOCK_RE.sub("", text)
+    return _DEEPSEEK_TAG_RE.sub("", text).strip()
+
+
+class DeepseekOcrEngine:
+    """DeepSeek-OCR over an OpenAI-compatible chat-completions endpoint.
+
+    One request per page, sequentially. The image travels as a base64 data
+    URL, which both Ollama and vLLM accept. Pages are sent one at a time
+    because the model server, not the worker, is the bottleneck; docling's
+    default of four concurrent requests and a 90 s timeout is what broke on
+    a laptop-hosted Ollama in testing.
+
+    Attributes:
+        url: Chat-completions URL (``OCR_DEEPSEEK_URL``).
+        model: Model name at that endpoint (``OCR_DEEPSEEK_MODEL``).
+        timeout: Seconds per page (``OCR_DEEPSEEK_TIMEOUT``).
+        prompt: Instruction sent with every page image.
+    """
+
+    name = "deepseek"
+
+    def __init__(
+        self,
+        url: Optional[str] = None,
+        model: Optional[str] = None,
+        timeout: Optional[float] = None,
+        prompt: str = DEEPSEEK_PROMPT,
+        max_tokens: int = _DEEPSEEK_MAX_TOKENS,
+    ) -> None:
+        from application.core.settings import settings
+
+        self.url = url or settings.OCR_DEEPSEEK_URL
+        self.model = model or settings.OCR_DEEPSEEK_MODEL
+        self.timeout = float(timeout if timeout is not None else getattr(settings, "OCR_DEEPSEEK_TIMEOUT", 300))
+        self.prompt = prompt
+        self.max_tokens = max_tokens
+
+    def payload(self, image) -> Dict:
+        """The chat-completions request body for one page image."""
+        data_url = "data:image/png;base64," + base64.b64encode(_png_bytes(image)).decode("ascii")
+        return {
+            "model": self.model,
+            "messages": [
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "image_url", "image_url": {"url": data_url}},
+                        {"type": "text", "text": self.prompt},
+                    ],
+                }
+            ],
+            "max_tokens": self.max_tokens,
+            "temperature": 0,
+        }
+
+    def ocr_image(self, image) -> str:
+        """OCR one PIL image through the endpoint.
+
+        Raises:
+            DocumentParseError: The request failed, timed out, or the
+                response was not a chat completion.
+        """
+        import requests
+
+        try:
+            response = requests.post(self.url, json=self.payload(image), timeout=self.timeout)
+            response.raise_for_status()
+            body = response.json()
+        except requests.exceptions.JSONDecodeError as exc:
+            # Subclasses RequestException too, so it must be caught first or a
+            # proxy's HTML error page is reported as a connection failure.
+            raise DocumentParseError(f"DeepSeek-OCR endpoint {self.url} returned a non-JSON body") from exc
+        except requests.RequestException as exc:
+            raise DocumentParseError(
+                f"DeepSeek-OCR request to {self.url} failed: {exc}. Check OCR_DEEPSEEK_URL "
+                f"and that model {self.model!r} is served (e.g. `ollama pull {self.model}`)."
+            ) from exc
+        try:
+            content = body["choices"][0]["message"]["content"]
+        except (KeyError, IndexError, TypeError) as exc:
+            raise DocumentParseError(
+                f"DeepSeek-OCR endpoint {self.url} returned no chat completion: {str(body)[:200]}"
+            ) from exc
+        if isinstance(content, list):  # some servers return content parts
+            content = "".join(part.get("text", "") for part in content if isinstance(part, dict))
+        return clean_deepseek_output(str(content or ""))
+
+
+def build_native_ocr_engine(engine: Optional[str] = None, languages: Optional[List[str]] = None) -> OcrEngine:
+    """Instantiate the engine ``OCR_ENGINE`` (or ``engine``) resolves to for the native backend."""
+    resolved = resolve_native_ocr_engine(engine)
+    if resolved == "deepseek":
+        return DeepseekOcrEngine()
+    return TesseractEngine(languages=languages)
+
+
+# ---------------------------------------------------------------------------
+# Parsers
+# ---------------------------------------------------------------------------
+
+
+def render_scale(page_width_pt: float, page_height_pt: float, dpi: int) -> float:
+    """Render scale for a page of the given size at ``dpi``, capped by ``_MAX_RENDER_PIXELS``.
+
+    The cap is what stands between an oversized (or hostile) MediaBox and a
+    multi-gigabyte bitmap allocation in the ingest worker.
+    """
+    scale = dpi / 72.0
+    area = max(1.0, float(page_width_pt)) * max(1.0, float(page_height_pt))
+    max_scale = (_MAX_RENDER_PIXELS / area) ** 0.5
+    return min(scale, max_scale)
+
+
+def _render_page(page, dpi: int):
+    """Render a pypdfium2 page to a PIL image at ``dpi``, within the pixel budget."""
+    width_pt, height_pt = page.get_size()
+    scale = render_scale(width_pt, height_pt, dpi)
+    if scale < dpi / 72.0:
+        logger.warning(
+            "Page is %.0fx%.0f pt; rendering at %.0f dpi instead of %d to stay within %d MP",
+            width_pt,
+            height_pt,
+            scale * 72.0,
+            dpi,
+            _MAX_RENDER_PIXELS // 1_000_000,
+        )
+    return page.render(scale=scale).to_pil()
+
+
+def text_layer_counts(path: Path) -> List[int]:
+    """Characters in each page's text layer (pypdfium2), in page order.
+
+    Raises:
+        Exception: Propagated from pypdfium2 when the file cannot be opened.
+    """
+    import pypdfium2 as pdfium
+
+    pdf = pdfium.PdfDocument(str(path))
+    counts: List[int] = []
+    try:
+        for index in range(len(pdf)):
+            page = pdf[index]
+            try:
+                textpage = page.get_textpage()
+                try:
+                    counts.append(max(0, textpage.count_chars()))
+                finally:
+                    textpage.close()
+            finally:
+                page.close()
+    finally:
+        pdf.close()
+    return counts
+
+
+def _page_has_image(page) -> bool:
+    """Whether the page draws at least one image XObject (something OCR could read)."""
+    import pypdfium2.raw as pdfium_c
+
+    try:
+        # pypdfium2's default max_depth=2 misses images inside nested Form
+        # XObjects, which print drivers and InDesign produce routinely.
+        objects = page.get_objects(filter=(pdfium_c.FPDF_PAGEOBJ_IMAGE,), max_depth=16)
+        return next(iter(objects), None) is not None
+    except Exception:  # noqa: BLE001 - treat an unreadable page as image-bearing
+        return True
+
+
+def scanned_page_indices(path: Path, min_chars: int = DEFAULT_TEXT_LAYER_MIN_CHARS) -> List[int]:
+    """0-based indices of the pages that look scanned: no text layer, or a thin one over an image.
+
+    The probe behind mixed-document handling: a text-layer converter (anydoc)
+    reads the text pages and silently skips these, so whoever owns OCR has to
+    fill them in. A page with no text at all always qualifies. A page with a
+    thin text layer (under ``min_chars``: a Bates stamp on a scan, a page
+    number under a full-page figure) qualifies only when it also draws an
+    image — a cover page reading just "Annual Report 2025" has nothing to OCR,
+    and OCR-ing it would duplicate text the converter already read. Returns []
+    when the file cannot be probed — the caller then keeps what it has rather
+    than failing a successful parse.
+    """
+    import pypdfium2 as pdfium
+
+    floor = max(1, int(min_chars))
+    indices: List[int] = []
+    try:
+        pdf = pdfium.PdfDocument(str(path))
+        try:
+            for index in range(len(pdf)):
+                page = pdf[index]
+                try:
+                    textpage = page.get_textpage()
+                    try:
+                        count = max(0, textpage.count_chars())
+                    finally:
+                        textpage.close()
+                    if count == 0 or (count < floor and _page_has_image(page)):
+                        indices.append(index)
+                finally:
+                    page.close()
+        finally:
+            pdf.close()
+    except Exception:  # noqa: BLE001 - a probe must never fail the parse
+        logger.warning("Could not probe %s for scanned pages", Path(path).name, exc_info=True)
+        return []
+    return indices
+
+
+def _check_near_empty(name: str, engine: str, content: str, pages: int, ocr_pages: int) -> None:
+    """Loud failure for OCR that produced nothing, a warning for text-sparse documents.
+
+    Mirrors the docling parser's dropout guard: failing only on positive
+    evidence — several pages that OCR'd to literally nothing — because a
+    photo, logo or chart is genuinely text-poor, and ``DocumentParseError``
+    rejects the upload permanently.
+    """
+    if ocr_pages == 0:
+        return
+    floor = ocr_min_chars_per_page()
+    chars = len(content.strip())
+    if floor <= 0 or chars >= floor * pages:
+        return
+    if chars == 0 and pages > 1:
+        raise DocumentParseError(
+            f"OCR ({engine}) produced no text over {pages} pages of {name}; "
+            "document not indexed. Check the engine's languages (OCR_LANGS) and "
+            "that the scan is legible."
+        )
+    logger.warning(
+        "%s OCR'd (%s) to only %d chars over %d page(s); indexing it as text-sparse",
+        name,
+        engine,
+        chars,
+        pages,
+    )
+
+
+class NativeOcrPdfParser(BaseParser):
+    """PDF parser that OCRs pages without a text layer and reads the rest directly.
+
+    Per page: a text layer of at least ``min_text_chars`` characters is read
+    with pypdfium2; anything thinner is rendered and sent to the engine. So
+    a scan, a born-digital PDF and a mix of both all come out right without
+    any prior classification.
+
+    Attributes:
+        engine: The ``OcrEngine``; built from ``OCR_ENGINE`` on first use when None.
+        text_parser: Optional parser for documents whose *every* page has a
+            text layer — used under ``DOC_PARSER_ENGINE=docling`` to keep
+            docling's structured Markdown for text PDFs while scans take the
+            native OCR path. None reads text layers with pypdfium2.
+        min_text_chars: Text-layer characters per page below which the page is OCR'd.
+        ocr_enabled: Always True; lets callers that inspect their fallback
+            parser (``AnydocParser``) phrase their errors correctly.
+        last_engine: Engine name behind the most recent parse.
+    """
+
+    ocr_enabled = True
+
+    def __init__(
+        self,
+        engine: Optional[OcrEngine] = None,
+        text_parser: Optional[BaseParser] = None,
+        min_text_chars: int = DEFAULT_TEXT_LAYER_MIN_CHARS,
+        parser_config: Optional[Dict] = None,
+    ) -> None:
+        super().__init__(parser_config)
+        self.engine = engine
+        self.text_parser = text_parser
+        self.min_text_chars = max(1, int(min_text_chars))
+        self.last_engine: Optional[str] = None
+        self._last_metadata: Dict = {}
+
+    def _init_parser(self) -> Dict:
+        try:
+            import pypdfium2  # noqa: F401
+            from PIL import Image  # noqa: F401
+        except ImportError as exc:
+            raise ImportError("pypdfium2 and Pillow are required for NativeOcrPdfParser") from exc
+        if self.engine is None:
+            self.engine = build_native_ocr_engine()
+        return {
+            "engine": self.engine.name,
+            "text_parser": type(self.text_parser).__name__ if self.text_parser else None,
+            "min_text_chars": self.min_text_chars,
+        }
+
+    def _ensure_engine(self) -> OcrEngine:
+        if self.engine is None:
+            self.engine = build_native_ocr_engine()
+        return self.engine
+
+    def _text_layer_counts(self, pdf) -> List[int]:
+        counts: List[int] = []
+        for index in range(len(pdf)):
+            page = pdf[index]
+            try:
+                textpage = page.get_textpage()
+                try:
+                    counts.append(max(0, textpage.count_chars()))
+                finally:
+                    textpage.close()
+            finally:
+                page.close()
+        return counts
+
+    def ocr_pages(self, file: Path, indices: List[int]) -> Dict[int, str]:
+        """OCR only the given 0-based pages of ``file``, ignoring their text layers.
+
+        Used by ``AnydocParser`` for mixed documents: anydoc has already read
+        the text pages, so only the scanned ones come here.
+
+        Returns:
+            Page index -> recognised text, in the order requested.
+
+        Raises:
+            DocumentParseError: The file cannot be opened or the engine failed.
+        """
+        import pypdfium2 as pdfium
+
+        path = Path(file)
+        engine = self._ensure_engine()
+        try:
+            pdf = pdfium.PdfDocument(str(path))
+        except Exception as exc:
+            raise DocumentParseError(f"Failed to open {path.name} with pypdfium2: {exc}") from exc
+        texts: Dict[int, str] = {}
+        try:
+            dpi = render_dpi()
+            for index in indices:
+                if index < 0 or index >= len(pdf):
+                    continue
+                page = pdf[index]
+                try:
+                    texts[index] = (engine.ocr_image(_render_page(page, dpi)) or "").strip()
+                finally:
+                    page.close()
+        except DocumentParseError:
+            raise
+        except Exception as exc:
+            raise DocumentParseError(f"Failed to OCR pages of {path.name} ({engine.name}): {exc}") from exc
+        finally:
+            pdf.close()
+        self.last_engine = engine.name
+        return texts
+
+    def text_layer_delegate(self, file: Path) -> Optional[BaseParser]:
+        """The ``text_parser`` that ``parse_file`` would hand ``file`` to, or None.
+
+        None when there is no text parser, the file cannot be opened, or any
+        page falls below ``min_text_chars`` (those are OCR'd here). Callers
+        that want a by-product of the delegate's conversion (docling tables)
+        can run it directly instead of converting the document twice.
+        """
+        if self.text_parser is None:
+            return None
+        try:
+            import pypdfium2 as pdfium
+
+            pdf = pdfium.PdfDocument(str(Path(file)))
+        except Exception:  # noqa: BLE001 - a probe; parse_file reports the real error
+            return None
+        try:
+            counts = self._text_layer_counts(pdf)
+        except Exception:  # noqa: BLE001
+            return None
+        finally:
+            pdf.close()
+        if counts and all(count >= self.min_text_chars for count in counts):
+            return self.text_parser
+        return None
+
+    def _delegate_text(self, path: Path, errors: str) -> Union[str, List[str]]:
+        text_parser = self.text_parser
+        assert text_parser is not None
+        logger.info(
+            "%s has a text layer on every page; parsing with %s instead of OCR",
+            path.name,
+            type(text_parser).__name__,
+        )
+        result = delegate_parse(text_parser, path, errors)
+        self.last_engine = getattr(text_parser, "last_engine", None) or type(text_parser).__name__
+        self._last_metadata = {**text_parser.get_file_metadata(path), "parse_engine": self.last_engine}
+        return result
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Parse a PDF, OCR-ing only the pages that need it.
+
+        Raises:
+            DocumentParseError: The file cannot be opened, the engine failed,
+                or several pages OCR'd to nothing.
+        """
+        import pypdfium2 as pdfium
+
+        path = Path(file)
+        engine = self._ensure_engine()
+        try:
+            pdf = pdfium.PdfDocument(str(path))
+        except Exception as exc:
+            raise DocumentParseError(f"Failed to open {path.name} with pypdfium2: {exc}") from exc
+
+        pages_text: List[str] = []
+        ocr_pages = 0
+        try:
+            page_count = len(pdf)
+            if page_count == 0:
+                raise DocumentParseError(f"{path.name} has no pages")
+            counts = self._text_layer_counts(pdf)
+            if self.text_parser is not None and all(count >= self.min_text_chars for count in counts):
+                pdf.close()
+                pdf = None
+                return self._delegate_text(path, errors)
+            dpi = render_dpi()
+            for index in range(page_count):
+                page = pdf[index]
+                try:
+                    if counts[index] >= self.min_text_chars:
+                        textpage = page.get_textpage()
+                        try:
+                            text = textpage.get_text_bounded()
+                        finally:
+                            textpage.close()
+                    else:
+                        text = engine.ocr_image(_render_page(page, dpi))
+                        ocr_pages += 1
+                finally:
+                    page.close()
+                pages_text.append((text or "").strip())
+        except DocumentParseError:
+            raise
+        except Exception as exc:
+            raise DocumentParseError(f"Failed to parse {path.name} with native OCR ({engine.name}): {exc}") from exc
+        finally:
+            if pdf is not None:
+                pdf.close()
+
+        content = "\n\n".join(pages_text)
+        _check_near_empty(path.name, engine.name, content, page_count, ocr_pages)
+        self.last_engine = engine.name
+        self._last_metadata = {"parse_engine": engine.name, "pdf_pages": page_count, "ocr_pages": ocr_pages}
+        logger.info(
+            "Parsed %s with native OCR (%s): %d/%d page(s) OCR'd, %d chars",
+            path.name,
+            engine.name,
+            ocr_pages,
+            page_count,
+            len(content),
+        )
+        return content
+
+    def get_file_metadata(self, file: Path) -> Dict:
+        """Engine and page counts for the most recently parsed file."""
+        _ = file
+        return dict(self._last_metadata)
+
+
+class NativeOcrImageParser(BaseParser):
+    """Image parser that OCRs every frame (multi-page TIFFs included) with the native engine."""
+
+    ocr_enabled = True
+
+    def __init__(self, engine: Optional[OcrEngine] = None, parser_config: Optional[Dict] = None) -> None:
+        super().__init__(parser_config)
+        self.engine = engine
+        self.last_engine: Optional[str] = None
+        self._last_metadata: Dict = {}
+
+    def _init_parser(self) -> Dict:
+        try:
+            from PIL import Image  # noqa: F401
+        except ImportError as exc:
+            raise ImportError("Pillow is required for NativeOcrImageParser") from exc
+        if self.engine is None:
+            self.engine = build_native_ocr_engine()
+        return {"engine": self.engine.name}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """OCR an image file.
+
+        Only TIFF frames are pages (multi-page faxes and scans); every other
+        multi-frame format is an animation (GIF, WebP, APNG) whose frames
+        repeat one picture, so only the first is OCR'd — a 50-frame WebP
+        would otherwise cost 50 tesseract runs and then be rejected as an
+        empty multi-page document.
+
+        Raises:
+            DocumentParseError: The image cannot be decoded, the engine failed,
+                or a multi-frame image OCR'd to nothing.
+        """
+        from PIL import Image, ImageSequence
+
+        _ = errors
+        path = Path(file)
+        if self.engine is None:
+            self.engine = build_native_ocr_engine()
+        engine = self.engine
+        texts: List[str] = []
+        try:
+            with Image.open(path) as image:
+                frames = ImageSequence.Iterator(image) if path.suffix.lower() in (".tif", ".tiff") else [image]
+                for frame in frames:
+                    texts.append((engine.ocr_image(fit_to_pixel_budget(frame)) or "").strip())
+        except DocumentParseError:
+            raise
+        except Exception as exc:
+            raise DocumentParseError(f"Failed to OCR {path.name} ({engine.name}): {exc}") from exc
+
+        content = "\n\n".join(texts)
+        _check_near_empty(path.name, engine.name, content, max(1, len(texts)), len(texts))
+        self.last_engine = engine.name
+        self._last_metadata = {"parse_engine": engine.name, "ocr_pages": len(texts)}
+        return content
+
+    def get_file_metadata(self, file: Path) -> Dict:
+        """Engine and frame count for the most recently parsed file."""
+        _ = file
+        return dict(self._last_metadata)

application/parser/file/pdf_trust.py

@@ -0,0 +1,280 @@
+"""Trust checks for PDF text extraction.
+
+anydoc silently drops text from fonts it cannot map to Unicode — seen with
+non-embedded composite (Type0/CID) fonts relying on predefined CMaps, where
+the EN/ZH NDA corpus file loses its whole Chinese column with no error, and
+with Adobe-CNS1 fonts in HK legal PDFs where a valid ToUnicode exists but
+extraction still fails. These dependency-free checks scan the PDF's raw
+objects (including Flate-compressed object streams) so the pipeline can
+route such files to a heavier parser, or at least mark the output as
+unverified, instead of trusting silent partial text.
+
+Two stages:
+
+* ``check_pdf_fonts`` — pre-flight on the bytes alone: composite fonts
+  without an embedded ToUnicode CMap, and whether the PDF declares CJK font
+  resources at all.
+* ``verify_extraction`` — pre-flight plus the cross-check the font scan
+  cannot do: a PDF that declares CJK fonts whose extracted text contains
+  almost no CJK characters is not to be trusted.
+
+A flag means "verify or route to a fallback", not "the text is wrong": tools
+shipping Adobe's predefined CMaps (docling's parser does) often extract such
+files fine. On the benchmark corpus the checks caught both known
+silent-drop cases with zero false positives on the other 14 PDFs, and cost
+~30 ms per scanned MB (92 ms on a 3 MB, 150-page annual report).
+"""
+import re
+import zlib
+from pathlib import Path
+from typing import Dict, Iterator, List, Optional, Union
+
+_OBJ_HEADER = re.compile(rb"\d+\s+\d+\s+obj")
+_TYPE0 = re.compile(rb"/Subtype\s*/Type0")
+# The CJK expectation is keyed on CIDSystemInfo /Ordering ALONE — the
+# authoritative "this PDF maps text through a CJK character collection"
+# signal, present in every known silent-drop case. Matching CJK font *names*
+# (SimSun, MS-Gothic, MSungHK, ...) was tried and rejected: Word-exported
+# Latin PDFs routinely embed an MS-Gothic subset for a stray full-width
+# character (a 150-page English annual report in the benchmark corpus does),
+# and substring matching also catches Latin faces like FranklinGothic —
+# either way English-only documents would be rerouted to the heavy engine.
+_CJK_ORDERING = re.compile(rb"/Ordering\s*\((GB1|CNS1|Japan1|Japan2|KR|Korea1)\)")
+# Bytes kept around a Type0 marker found inside a decompressed object
+# stream: object streams hold many dicts with no obj/endobj markers, and a
+# font dict's keys sit close to its /Subtype entry.
+_WINDOW_BEFORE, _WINDOW_AFTER = 200, 800
+
+# Extracted text with fewer CJK characters than this, from a PDF that
+# declares CJK fonts, is treated as a silent drop.
+_MIN_CJK_CHARS = 10
+_CJK_PROBLEM_PREFIX = "PDF declares CJK fonts"
+
+_CJK_RANGES = (
+    ("一", "鿿"),  # CJK Unified Ideographs
+    ("぀", "ヿ"),  # Hiragana + Katakana
+    ("가", "힯"),  # Hangul syllables
+)
+
+
+# Per-stream decompression cap. Flate reaches ~1000:1, so uncapped inflation
+# of a crafted (or merely image-heavy) PDF inside the upload cap could balloon
+# to gigabytes and OOM the ingest worker. Font dicts and object streams — the
+# only things the signals live in — are far smaller than this.
+_STREAM_INFLATE_CAP = 4_000_000
+
+
+def _stream_bodies(raw: bytes) -> Iterator[bytes]:
+    """The raw bytes between each ``stream`` / ``endstream`` keyword pair.
+
+    A ``find`` walk rather than a regex: ``stream\\r?\\n(.*?)\\r?\\nendstream``
+    misses every stream whose data runs straight into ``endstream`` with no
+    EOL (common in the wild), and when it misses one its lazy group scans on
+    to the *next* stream's terminator — quadratic on large files (a 10 MB
+    file measured 32 s, matching a fifth of its streams) and a merged,
+    undecodable body for the rest.
+    """
+    pos = 0
+    while True:
+        pos = raw.find(b"stream", pos)
+        if pos < 0:
+            return
+        if raw[max(0, pos - 3):pos] == b"end":
+            pos += 6
+            continue
+        start = pos + 6
+        if raw[start:start + 2] == b"\r\n":
+            start += 2
+        elif raw[start:start + 1] in (b"\n", b"\r"):
+            start += 1
+        end = raw.find(b"endstream", start)
+        if end < 0:
+            return
+        body = raw[start:end]
+        if body.endswith(b"\r\n"):
+            body = body[:-2]
+        elif body.endswith((b"\n", b"\r")):
+            body = body[:-1]
+        pos = end + 9
+        yield body
+
+
+def _object_bodies(raw: bytes) -> Iterator[bytes]:
+    """The bytes between each ``N G obj`` header and the next ``endobj``.
+
+    A forward walk for the same reason as ``_stream_bodies``: the regex
+    ``\\d+\\s+\\d+\\s+obj(.*?)endobj`` rescans to end-of-file for every header
+    that has no ``endobj`` after it, so a malformed (or crafted) PDF with many
+    such headers costs headers x size — 2000 orphan headers in 1 MB measured
+    13.5 s, on the default upload path. Advancing past each ``endobj`` keeps
+    the walk linear and yields the same bodies.
+    """
+    pos = 0
+    while True:
+        header = _OBJ_HEADER.search(raw, pos)
+        if header is None:
+            return
+        end = raw.find(b"endobj", header.end())
+        if end < 0:
+            return
+        yield raw[header.end():end]
+        pos = end + 6
+
+
+def _decompressed_streams(raw: bytes) -> Iterator[bytes]:
+    """Each Flate stream in ``raw`` that decompresses, one at a time, capped."""
+    for body in _stream_bodies(raw):
+        try:
+            inflated = zlib.decompressobj().decompress(body, _STREAM_INFLATE_CAP)
+        except zlib.error:
+            continue
+        yield inflated
+
+
+def _cjk_chars(text: str, up_to: int) -> int:
+    """Count CJK characters in ``text``, stopping once ``up_to`` is reached."""
+    count = 0
+    for char in text:
+        for low, high in _CJK_RANGES:
+            if low <= char <= high:
+                count += 1
+                break
+        if count >= up_to:
+            break
+    return count
+
+
+def check_pdf_fonts(data: bytes) -> Dict[str, Union[int, bool]]:
+    """Pre-flight font scan of a PDF's bytes.
+
+    Args:
+        data: The complete PDF file contents.
+
+    Returns:
+        Dict with ``type0`` (composite fonts seen), ``type0_no_tounicode``
+        (those without an embedded ToUnicode CMap), ``expects_cjk`` (the PDF
+        declares CJK font resources), ``has_fonts``, and ``flagged`` — True
+        when extraction should not be trusted unverified.
+    """
+    type0 = bare = 0
+    expects_cjk = bool(_CJK_ORDERING.search(data))
+    has_fonts = b"/Font" in data
+
+    def count_font(unit: bytes) -> None:
+        """One analysis unit holding a Type0 font dict."""
+        nonlocal type0, bare
+        type0 += 1
+        if b"/ToUnicode" not in unit:
+            bare += 1
+
+    for body in _object_bodies(data):
+        if _TYPE0.search(body):
+            count_font(body)
+    # Each decompressed stream is scanned for every signal in one pass and
+    # then dropped; nothing inflated is retained past its loop iteration.
+    for stream in _decompressed_streams(data):
+        expects_cjk = expects_cjk or bool(_CJK_ORDERING.search(stream))
+        has_fonts = has_fonts or b"/Font" in stream
+        for type0_match in _TYPE0.finditer(stream):
+            start = type0_match.start()
+            count_font(stream[max(0, start - _WINDOW_BEFORE): start + _WINDOW_AFTER])
+    return {
+        "type0": type0,
+        "type0_no_tounicode": bare,
+        "expects_cjk": expects_cjk,
+        "has_fonts": has_fonts,
+        "flagged": bare > 0 or not has_fonts,
+    }
+
+
+def verify_extraction(data: bytes, markdown: str) -> List[str]:
+    """Reasons the extracted ``markdown`` of the PDF ``data`` shouldn't be trusted.
+
+    Combines the pre-flight font scan with the post-conversion cross-check it
+    cannot do alone: fonts with a valid ToUnicode that the converter
+    nevertheless failed to extract still show up as missing CJK output.
+
+    Args:
+        data: The complete PDF file contents.
+        markdown: The text a converter extracted from it.
+
+    Returns:
+        Human-readable problem strings; empty when the output looks sound.
+    """
+    problems: List[str] = []
+    result = check_pdf_fonts(data)
+    if result["type0_no_tounicode"]:
+        problems.append(
+            f"{result['type0_no_tounicode']} composite (Type0) font(s) carry no "
+            "ToUnicode map; extracted text may silently omit glyphs"
+        )
+    elif not result["has_fonts"]:
+        problems.append(
+            "no font resources detected; text extraction cannot be verified"
+        )
+    if result["expects_cjk"]:
+        cjk = _cjk_chars(markdown, _MIN_CJK_CHARS)
+        if cjk < _MIN_CJK_CHARS:
+            problems.append(
+                f"{_CJK_PROBLEM_PREFIX} but the extracted text contains only "
+                f"{cjk} CJK character(s)"
+            )
+    return problems
+
+
+def _text_layer_cjk_chars(file: Path, up_to: int) -> Optional[int]:
+    """CJK characters in the text layer as pypdfium2 reads it, stopping at ``up_to``.
+
+    pdfium ships Adobe's predefined CMaps, so it is an independent reference
+    for whether the document *has* CJK text to lose. None when pypdfium2 is
+    unavailable or the file cannot be read that way.
+    """
+    try:
+        import pypdfium2 as pdfium
+    except ImportError:
+        return None
+    try:
+        pdf = pdfium.PdfDocument(str(file))
+    except Exception:  # noqa: BLE001 - a reference probe, not a parse
+        return None
+    count = 0
+    try:
+        for index in range(len(pdf)):
+            page = pdf[index]
+            try:
+                textpage = page.get_textpage()
+                try:
+                    text = textpage.get_text_bounded()
+                finally:
+                    textpage.close()
+            finally:
+                page.close()
+            count += _cjk_chars(text, up_to - count)
+            if count >= up_to:
+                break
+    except Exception:  # noqa: BLE001
+        return None
+    finally:
+        pdf.close()
+    return count
+
+
+def verify_pdf_file(file: Path, markdown: str) -> List[str]:
+    """``verify_extraction`` for a file on disk; unreadable files trust the output.
+
+    The CJK cross-check is confirmed against pdfium's own text layer: a
+    stray ``/Ordering (Japan1)`` in an otherwise Latin document (a 48-page
+    Quartz export in the corpus carries one among 258 Identity fonts) is not
+    a dropped Chinese column, and must not send the file to a full docling
+    re-parse.
+    """
+    try:
+        data = Path(file).read_bytes()
+    except OSError:
+        return []
+    problems = verify_extraction(data, markdown)
+    if any(problem.startswith(_CJK_PROBLEM_PREFIX) for problem in problems):
+        reference = _text_layer_cjk_chars(Path(file), _MIN_CJK_CHARS)
+        if reference is not None and reference < _MIN_CJK_CHARS:
+            problems = [p for p in problems if not p.startswith(_CJK_PROBLEM_PREFIX)]
+    return problems

application/parser/file/tableize.py

@@ -0,0 +1,105 @@
+"""Reconstruct typographic tables in anydoc's PDF Markdown output.
+
+anydoc converts PDFs straight to Markdown with no document model, so tables
+drawn with dot leaders or bare whitespace alignment (financial statements,
+tables of contents) come out as flat text lines — values intact, structure
+lost. This post-processor detects runs of such lines and rewrites them as
+GFM tables: the lightweight alternative to a table-structure model for this
+layout family. (The robust fix is upstream in anydoc's Rust PDF code, where
+glyph x-positions exist; this is the recoverable-downstream version.)
+
+Deliberately conservative — a run converts only when it has at least
+``min_rows`` consecutive lines that each parse as ``label [leaders]
+numeric-columns`` with the *same* column count; anything else passes through
+untouched. Gated by ``ANYDOC_TABLEIZE`` (off by default) and applied only to
+anydoc's own PDF output, never to docling's.
+"""
+import re
+from typing import List, Optional, Tuple
+
+# label ...... 1,234 (56) — dot-leader row
+_LEADER = re.compile(r"^(.*?)\s*\.{3,}\s*(.+)$")
+# numeric-ish token: $ 1,234 / (30) / 83,431 / 12.5% / —
+_NUM_TOKEN = re.compile(r"\$?\(?-?\d[\d,.]*\)?%?|—")
+# row without leaders: label text, then a trailing run of numeric tokens
+_TRAILING = re.compile(r"^(.*?[^\s\d,.])\s+([\d$(].*)$")
+
+
+def _merge_currency(tokens: List[str]) -> List[str]:
+    """Join a free-standing ``$`` onto the number that follows it."""
+    out: List[str] = []
+    for token in tokens:
+        if out and out[-1] == "$":
+            out[-1] = "$" + token
+        else:
+            out.append(token)
+    return out
+
+
+def _parse_row(line: str) -> Optional[Tuple[str, List[str]]]:
+    """``(label, values)`` when the line looks like a typographic table row, else None.
+
+    Without dot leaders a row needs at least two numeric columns: a single
+    trailing number is what "Chapter 1", "Footnote 2", "ISO 9001" and
+    "Version 3.0" look like, and three of those in a row are a list, not a
+    table.
+    """
+    leader = _LEADER.match(line)
+    match = leader or _TRAILING.match(line)
+    if not match:
+        return None
+    label, rest = match.group(1).strip(" ."), match.group(2)
+    values = _merge_currency(rest.split())
+    if not values or not all(_NUM_TOKEN.fullmatch(v.lstrip("$")) for v in values):
+        return None
+    if leader is None and len(values) < 2:
+        return None
+    if not label or _NUM_TOKEN.fullmatch(label):
+        return None
+    return label, values
+
+
+def tableize(markdown: str, min_rows: int = 3) -> str:
+    """Rewrite runs of typographic table rows in ``markdown`` as GFM tables.
+
+    Args:
+        markdown: Converter output to post-process.
+        min_rows: Minimum consecutive, same-width rows for a run to convert.
+
+    Returns:
+        ``markdown`` with qualifying runs rewritten; everything else verbatim.
+    """
+    out: List[str] = []
+    run: List[Tuple[str, List[str], str]] = []  # (label, values, original line)
+
+    def flush() -> None:
+        nonlocal run
+        widths = {len(values) for _, values, _ in run}
+        if len(run) >= min_rows and len(widths) == 1:
+            ncols = widths.pop()
+            out.append("")
+            # GFM needs a header row; an empty one keeps invented labels out
+            # of the indexed text (they would otherwise be embedded as content).
+            out.append("|" + "   |" * (ncols + 1))
+            out.append("|" + " --- |" * (ncols + 1))
+            for label, values, _ in run:
+                # Only the label can carry a '|' (values are numeric tokens);
+                # unescaped it would add a cell and mis-column the row.
+                cells = [label.replace("|", "\\|")] + values
+                out.append("| " + " | ".join(cells) + " |")
+            out.append("")
+        else:
+            out.extend(original for _, _, original in run)
+        run = []
+
+    for line in markdown.splitlines():
+        parsed = _parse_row(line.strip()) if line.strip() else None
+        if parsed:
+            run.append((*parsed, line))
+        else:
+            if run:
+                flush()
+            out.append(line)
+    if run:
+        flush()
+    return "\n".join(out)

application/parser/remote/crawler_markdown.py

@@ -110,8 +110,9 @@ class CrawlerLoader(BaseRemote):
         language = language_tag.get('lang', 'en') if language_tag else "en"
 
         markdownified = markdownify(html_content, heading_style="ATX", newline_style="BACKSLASH")
-        # Reduce sequences of more than two newlines to exactly three
-        markdownified = re.sub(r'\n{3,}', '\n\n\n', markdownified)
+        # Collapse runs of blank lines to a single one — the same shape
+        # ``html_to_markdown`` gives uploaded HTML files.
+        markdownified = re.sub(r'\n{3,}', '\n\n', markdownified)
         return title, language, markdownified
 
     def _extract_links(self, html_content, current_url):

application/parser/remote/s3_loader.py

@@ -5,6 +5,7 @@ import tempfile
 import mimetypes
 from typing import List, Optional
 from application.core.url_validation import SSRFError, validate_url
+from application.parser.file.constants import SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS
 from application.parser.remote.base import BaseRemote
 from application.parser.schema.base import Document
 
@@ -17,6 +18,61 @@ except ImportError:
 logger = logging.getLogger(__name__)
 
 
+# Plain-text suffixes the loader reads directly (no document parser).
+_TEXT_EXTENSIONS = {
+        ".txt",
+        ".md",
+        ".markdown",
+        ".rst",
+        ".json",
+        ".xml",
+        ".yaml",
+        ".yml",
+        ".py",
+        ".js",
+        ".ts",
+        ".jsx",
+        ".tsx",
+        ".java",
+        ".c",
+        ".cpp",
+        ".h",
+        ".hpp",
+        ".cs",
+        ".go",
+        ".rs",
+        ".rb",
+        ".php",
+        ".swift",
+        ".kt",
+        ".scala",
+        ".html",
+        ".css",
+        ".scss",
+        ".sass",
+        ".less",
+        ".sh",
+        ".bash",
+        ".zsh",
+        ".fish",
+        ".sql",
+        ".r",
+        ".m",
+        ".mat",
+        ".ini",
+        ".cfg",
+        ".conf",
+        ".config",
+        ".env",
+        ".gitignore",
+        ".dockerignore",
+        ".editorconfig",
+        ".log",
+        ".csv",
+        ".tsv",
+    }
+
+
 class S3Loader(BaseRemote):
     """Load documents from an AWS S3 bucket."""
 
@@ -128,58 +184,7 @@ class S3Loader(BaseRemote):
 
     def is_text_file(self, file_path: str) -> bool:
         """Determine if a file is a text file based on extension."""
-        text_extensions = {
-            ".txt",
-            ".md",
-            ".markdown",
-            ".rst",
-            ".json",
-            ".xml",
-            ".yaml",
-            ".yml",
-            ".py",
-            ".js",
-            ".ts",
-            ".jsx",
-            ".tsx",
-            ".java",
-            ".c",
-            ".cpp",
-            ".h",
-            ".hpp",
-            ".cs",
-            ".go",
-            ".rs",
-            ".rb",
-            ".php",
-            ".swift",
-            ".kt",
-            ".scala",
-            ".html",
-            ".css",
-            ".scss",
-            ".sass",
-            ".less",
-            ".sh",
-            ".bash",
-            ".zsh",
-            ".fish",
-            ".sql",
-            ".r",
-            ".m",
-            ".mat",
-            ".ini",
-            ".cfg",
-            ".conf",
-            ".config",
-            ".env",
-            ".gitignore",
-            ".dockerignore",
-            ".editorconfig",
-            ".log",
-            ".csv",
-            ".tsv",
-        }
+        text_extensions = _TEXT_EXTENSIONS
 
         file_lower = file_path.lower()
         for ext in text_extensions:
@@ -197,7 +202,9 @@ class S3Loader(BaseRemote):
 
     def is_supported_document(self, file_path: str) -> bool:
         """Check if file is a supported document type for parsing."""
-        document_extensions = {
+        # The upload whitelist plus the historical S3 list, so an object the
+        # upload API accepts is never silently skipped here.
+        document_extensions = (set(SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS) - _TEXT_EXTENSIONS) | {
             ".pdf",
             ".docx",
             ".doc",

application/requirements-docling.txt

@@ -0,0 +1,15 @@
+# Optional docling parser engine.
+#
+# Needed only for: DOC_PARSER_ENGINE=docling, the docling OCR backend
+# (OCR_BACKEND=docling: layout-model hybrid OCR, ocrmac/rapidocr engines),
+# .adoc/.vtt/.xml attachment parsing, and the read_document tool's `structured`
+# output. The default anydoc engine needs none of this, and OCR itself does
+# not either: OCR_ENABLED=true with the tesseract binary (or a DeepSeek-OCR
+# endpoint) runs through application/parser/file/ocr_parser.py.
+#
+# Install ON TOP of requirements.txt, which pins torch, transformers and
+# onnxruntime (rapidocr runs on the same onnxruntime fastembed uses):
+#   pip install -r requirements.txt -r requirements-docling.txt
+# Docker: build with --build-arg INSTALL_DOCLING=true
+docling==2.119.0
+rapidocr==3.9.2

application/requirements.txt

@@ -12,15 +12,19 @@ cryptography==50.0.0
 dataclasses-json==0.6.7
 daytona==0.205.1
 defusedxml==0.7.1
-docling==2.119.0
-rapidocr==3.9.2
-onnxruntime==1.28.0
+# Default document converter (DOC_PARSER_ENGINE=anydoc). The docling engine
+# (OCR, layout/table models, read_document's structured output) is an
+# optional extra: pip install -r application/requirements-docling.txt
+firecrawl-anydoc==0.2.3
 docx2txt==0.9
 ddgs>=8.0.0
 fast-ebook
 elevenlabs==2.62.0
 Flask==3.1.3
 fastembed==0.8.0
+# fastembed's runtime: local embeddings execute on it, so it stays pinned in
+# core even though the docling extra's rapidocr runs on the same package.
+onnxruntime==1.28.0
 faiss-cpu==1.15.0
 fastmcp==3.4.6
 flask-restx==1.3.2
@@ -95,10 +99,12 @@ sqlalchemy>=2.0,<3
 starlette>=1.0,<2
 tiktoken==0.13.0
 tokenizers==0.22.2
+# torch and transformers serve docling's model stack alone -- embeddings moved
+# to fastembed/onnxruntime. Still in core rather than requirements-docling.txt,
+# so installing the optional extra never re-resolves the shared pins; both can
+# move out with docling.
 torch==2.11.0
 tqdm==4.67.3
-# torch and transformers are here for docling's model stack only -- embeddings
-# moved to fastembed/onnxruntime. Both can go when docling does.
 # Capped <5.9.0 to match docling-core's own darwin pin: 5.9+ breaks docling's
 # PDF layout model on Apple Silicon ("Cannot convert a MPS Tensor to float64").
 transformers==5.8.1

application/worker.py

@@ -1767,12 +1767,13 @@ def attachment_worker(self, file_info, user):
             scope={"kind": "attachment", "id": str(attachment_id)},
         )
 
-        # Attachments only: PDFs are read via their text layer where one
-        # exists (see ``ATTACHMENT_PDF_TEXT_FAST_PATH``). Source ingestion
+        # Attachments only: under the docling engine PDFs are read via their
+        # text layer where one exists (``ATTACHMENT_PDF_TEXT_FAST_PATH``; a
+        # no-op under anydoc, which is already that fast). Source ingestion
         # calls SimpleDirectoryReader without a ``file_extractor`` and so keeps
-        # the docling default, which is what retrieval quality depends on.
+        # the ``DOC_PARSER_ENGINE`` default map, which retrieval quality depends on.
         file_extractor = get_default_file_extractor(
-            ocr_enabled=settings.DOCLING_OCR_ATTACHMENTS_ENABLED,
+            ocr_enabled=settings.OCR_ATTACHMENTS_ENABLED,
             pdf_text_fast_path=settings.ATTACHMENT_PDF_TEXT_FAST_PATH,
         )
         _parser = file_extractor.get(os.path.splitext(filename)[1].lower())
@@ -1805,7 +1806,7 @@ def attachment_worker(self, file_info, user):
         parser_metadata = {
             key: value
             for key, value in (attachment_document.extra_info or {}).items()
-            if key.startswith("transcript_")
+            if key.startswith("transcript_") or key == "parse_warnings"
         }
         if parser_metadata:
             metadata = {**metadata, **parser_metadata}

deployment/docker-compose-azure.yaml

@@ -10,7 +10,13 @@ services:
       - backend
 
   backend:
-    build: ../application
+    build:
+      context: ../application
+      args:
+        INSTALL_DOCLING: ${INSTALL_DOCLING:-false}
+        # Off by default; deployments running OCR_ENABLED=true with tesseract
+        # must set INSTALL_TESSERACT=true before rebuilding (see docker-compose.yaml).
+        INSTALL_TESSERACT: ${INSTALL_TESSERACT:-false}
     env_file:
       - ../.env
     environment:
@@ -31,7 +37,13 @@ services:
         condition: service_healthy
 
   worker:
-    build: ../application
+    build:
+      context: ../application
+      args:
+        INSTALL_DOCLING: ${INSTALL_DOCLING:-false}
+        # Off by default; deployments running OCR_ENABLED=true with tesseract
+        # must set INSTALL_TESSERACT=true before rebuilding (see docker-compose.yaml).
+        INSTALL_TESSERACT: ${INSTALL_TESSERACT:-false}
     # `parsing` queue carries read_document/parse_document; required for its await to resolve.
     command: celery -A application.app.celery worker -l INFO -Q docsgpt,parsing,embeddings
     env_file:

deployment/docker-compose.yaml

@@ -15,7 +15,19 @@ services:
 
   backend:
     user: root
-    build: ../application
+    build:
+      context: ../application
+      args:
+        # Bake the optional docling engine (layout-model OCR backend, structured
+        # output) into the image: set INSTALL_DOCLING=true in ../.env or the shell.
+        INSTALL_DOCLING: ${INSTALL_DOCLING:-false}
+        # Bake the tesseract binary behind OCR_ENABLED=true (~35 MB): set
+        # INSTALL_TESSERACT=true in ../.env or the shell (setup.sh does this
+        # when OCR is enabled). Off by default, like docling. Upgrading a
+        # deployment that already runs OCR_ENABLED=true with tesseract: add
+        # INSTALL_TESSERACT=true to ../.env before rebuilding, or scanned
+        # pages fail with an install hint.
+        INSTALL_TESSERACT: ${INSTALL_TESSERACT:-false}
     env_file:
       - ../.env
     environment:
@@ -38,7 +50,11 @@ services:
 
   worker:
     user: root
-    build: ../application
+    build:
+      context: ../application
+      args:
+        INSTALL_DOCLING: ${INSTALL_DOCLING:-false}
+        INSTALL_TESSERACT: ${INSTALL_TESSERACT:-false}
     # Consumes the default queue AND the dedicated `parsing` (read_document /
     # parse_document) and `embeddings` (query embedding) queues. Without `parsing`
     # the read_document await never resolves; without `embeddings` every search

deployment/sandbox/README.md

@@ -188,8 +188,10 @@ Daytona render output matches the Jupyter-backend output.
 
 Document reading no longer runs in this sandbox. The `read_document` tool and the
 workflow native-file extract branch enqueue a `parse_document` Celery task that
-parses the document **in the backend** (Docling, already in
-`application/requirements.txt`) and awaits the result. The task is routed to a
+parses the document **in the backend** (the `DOC_PARSER_ENGINE` parser — anydoc
+by default; Docling only when the optional
+`application/requirements-docling.txt` extra is installed) and awaits the
+result. The task is routed to a
 dedicated **`parsing` queue** (`settings.DOCUMENT_PARSE_QUEUE`, default
 `"parsing"`) so a parse enqueued from inside a Celery worker (headless/scheduled
 agent) is served by a separate worker and never self-deadlocks the awaiting one.
@@ -200,8 +202,13 @@ Run a dedicated parsing worker that consumes the `parsing` queue:
 celery -A application.app.celery worker -Q parsing -l INFO
 ```
 
-It can be GPU-enabled with its own env (`DOCLING_OCR_ENABLED=true` plus GPU
-libraries) so OCR-heavy parsing runs on a separate, optionally larger pool.
+It takes its own env, so parse-heavy work runs on a separate, optionally larger
+pool. A GPU helps it only with the docling extra installed
+(`INSTALL_DOCLING=true`), whose layout and table models run on torch:
+`OCR_ENABLED=true` alone keeps the CPU-only native backend with the default
+`OCR_ENGINE=tesseract`, which GPU libraries do not accelerate. Setting
+`OCR_ENGINE=deepseek` instead moves the OCR cost onto the Ollama/vLLM endpoint
+and leaves this worker light.
 
 **Dev / single-worker setups:** without a dedicated parsing worker the default
 worker must also consume `parsing`, or the tool's await never resolves:

docs/content/Deploying/Development-Environment.mdx

@@ -94,6 +94,13 @@ To run the DocsGPT backend locally, you'll need to set up a Python environment a
     pip install -r application/requirements.txt
     ```
 
+    Optionally add the docling parser engine (OCR, `structured` output for
+    `read_document`; the default anydoc engine does not need it):
+
+    ```bash
+    pip install -r application/requirements-docling.txt
+    ```
+
 5.  **Run the Backend:**
 
     For local development, run the ASGI composition under uvicorn. It serves the **whole** application, hot-reloads on source changes, and matches the production runtime:

docs/content/Deploying/Docker-Deploying.mdx

@@ -48,7 +48,7 @@ The fastest way to try out DocsGPT is by using the public API endpoint. This req
     Navigate to the root directory of the DocsGPT repository in your terminal and run:
 
     ```bash
-    docker compose -f deployment/docker-compose.yaml up -d
+    docker compose --env-file .env -f deployment/docker-compose.yaml up -d
     ```
 
     The `-d` flag runs Docker Compose in detached mode (in the background).

docs/content/Deploying/DocsGPT-Settings.mdx

@@ -199,6 +199,32 @@ These byte/count limits and the non-negative nesting depth can be overridden in
 The request limit intentionally applies only to public upload routes; internal
 worker index transfers use their own trusted service boundary.
 
+## Document Parsing
+
+Uploaded sources and chat attachments are converted to Markdown before
+chunking or prompting. See the [Document Parsing and OCR](/Guides/ocr) guide
+for the engines and flows.
+
+| Setting | Default | Description |
+| --- | --- | --- |
+| `DOC_PARSER_ENGINE` | `anydoc` | Parser engine: `anydoc` (fast Rust converter, no ML models) or `docling` (layout/table models, structured output). Files anydoc cannot read fall back to Docling when installed. Docling is an optional extra: `pip install -r application/requirements-docling.txt`, or Docker builds with `--build-arg INSTALL_DOCLING=true`. |
+| `OCR_ENABLED` | `false` | OCR for source ingestion. Alias: `DOCLING_OCR_ENABLED`. |
+| `OCR_ATTACHMENTS_ENABLED` | `false` | OCR for chat attachments. Alias: `DOCLING_OCR_ATTACHMENTS_ENABLED`. |
+| `OCR_BACKEND` | `auto` | Who performs OCR: `auto` (Docling when installed, else native), `native` (pypdfium2 + Pillow rendering into tesseract or DeepSeek-OCR; no docling needed), or `docling` (layout-model hybrid OCR). See the [OCR guide](/Guides/ocr#ocr-backends). |
+| `ATTACHMENT_PDF_TEXT_FAST_PATH` | `true` | Under the `docling` engine, read PDF attachments through their text layer (pypdfium2) and only hand scans to Docling. Not needed under `anydoc`. |
+| `DOCLING_TABULAR_MAX_BYTES` | `2000000` | Docling engine: CSV/XLSX larger than this (by content) use the lightweight tabular parsers. |
+| `DOCLING_MARKUP_MAX_BYTES` | `8000000` | Docling engine: HTML/VTT larger than this are head-truncated before parsing. |
+| `MARKUP_MAX_BYTES` | `8000000` | anydoc engine: HTML/XHTML larger than this are head-truncated before the `markdownify` parser runs (its tree costs ~50x the input). `0` disables the gate. |
+| `PDF_TRUST_CHECK` | `true` | Trust-check anydoc's PDF output for the two silent-loss classes (Type0 fonts without ToUnicode, CJK-declaring PDFs with CJK-less text). Flagged files re-parse on Docling when installed, else carry `parse_warnings` metadata. |
+| `ANYDOC_TABLEIZE` | `false` | Rewrite dot-leader / whitespace-aligned table runs in anydoc's PDF markdown into GFM tables. |
+| `OCR_ENGINE` | `tesseract` | OCR engine when OCR is on: `tesseract` (recommended, both backends; optional system package — build with `INSTALL_TESSERACT=true` or `apt`/`brew` install it), `deepseek` (VLM endpoint, both backends), or the Docling-only `auto`, `ocrmac`, `rapidocr`. Unavailable engines degrade with a warning. See the [OCR guide](/Guides/ocr). |
+| `OCR_LANGS` | `eng` | Tesseract language packs, `+`-separated (e.g. `eng+chi_sim+deu`). |
+| `OCR_DEEPSEEK_URL` | `http://localhost:11434/v1/chat/completions` | OpenAI-compatible endpoint for `OCR_ENGINE=deepseek` (Ollama or vLLM). |
+| `OCR_DEEPSEEK_MODEL` | `deepseek-ocr:3b` | Model name at that endpoint. |
+| `OCR_DEEPSEEK_TIMEOUT` | `300` | Seconds allowed per page request to the DeepSeek endpoint, on both backends (the native backend sends pages one at a time). |
+| `OCR_RENDER_DPI` | `200` | Native backend: resolution at which pages without a text layer are rendered before OCR (clamped to 72-600). |
+| `OCR_MIN_CHARS_PER_PAGE` | `20` | Chars-per-page floor for the OCR dropout guard: a multi-page parse whose OCR returns nothing fails loudly instead of indexing an empty document (docling retries once on full-page OCR first); output below the floor but non-empty is indexed with a warning. `0` disables. Alias: `DOCLING_OCR_MIN_CHARS_PER_PAGE`. |
+
 ## Speech-to-Text Settings
 
 DocsGPT can transcribe audio in two places:

docs/content/Guides/How-to-train-on-other-documentation.mdx

@@ -16,7 +16,7 @@ Training on other documentation sources can greatly enhance the versatility and
 Make sure you have the document on which you want to train on  ready with you on the device which you are using .You can also use links to the documentation to train on.
 
 <Callout type="warning" emoji="⚠️">
- Note: Supported file formats include .pdf, .txt, .rst, .docx, .md, .mdx, .csv, .epub, .html, .json, .xlsx, .pptx, .png, .jpg, .jpeg, and audio files (.wav, .mp3, .m4a, .ogg, .webm). You can also train using the link of the documentation.
+ Note: Supported file formats include .pdf, .txt, .rst, .md, .mdx, .html, .xhtml, .json, .epub, Office documents (.docx, .doc, .docm, .odt, .rtf), spreadsheets (.xlsx, .xls, .xlsm, .xlsb, .ods, .csv), presentations (.pptx, .ppt, .pptm, .pps, .ppsx, .ppsm, .pot, .odp), images (.png, .jpg, .jpeg), and audio files (.wav, .mp3, .m4a, .ogg, .webm). You can also train using the link of the documentation.
 
 </Callout>
 

docs/content/Guides/ocr.mdx

@@ -1,21 +1,245 @@
 ---
-title: OCR for Sources and Attachments
-description: How OCR works in DocsGPT, how to configure it, and what changes for source ingestion vs chat attachments.
+title: Document Parsing and OCR
+description: Which engine parses uploaded documents, how OCR works in DocsGPT, how to configure it, and what changes for source ingestion vs chat attachments.
 ---
 
 import { Callout } from 'nextra/components'
 
-# Docling OCR for Sources and Attachments
+# Document Parsing and OCR for Sources and Attachments
 
-DocsGPT uses Docling as the default parser layer for many document formats. OCR is optional and controlled by two settings:
+## Parser engine
+
+DocsGPT converts uploaded documents to Markdown before chunking, embedding or
+handing them to a model. The engine is selected with one setting:
 
 ```env
-DOCLING_OCR_ENABLED=false
-DOCLING_OCR_ATTACHMENTS_ENABLED=false
+DOC_PARSER_ENGINE=anydoc
 ```
 
-- `DOCLING_OCR_ENABLED`: OCR behavior for Source Docs ingestion.
-- `DOCLING_OCR_ATTACHMENTS_ENABLED`: OCR behavior for chat attachments uploaded from the message box.
+- `anydoc` (default): [firecrawl-anydoc](https://github.com/firecrawl/anydoc),
+  a Rust converter with no ML models. It reads PDF, DOCX, PPTX, XLSX and CSV
+  in milliseconds with ~100 MB peak memory; HTML/XHTML is converted with
+  `markdownify`, head-truncated at `MARKUP_MAX_BYTES`. Because anydoc is a
+  core dependency, source uploads and the `read_document` tool now also
+  accept its other formats — DOC, PPT/PPS/POT, XLS, ODT/ODS/ODP, RTF, XHTML
+  and the macro-enabled Office variants — on every engine (they are in the
+  `SUPPORTED_SOURCE_EXTENSIONS` whitelist; chat attachments are not
+  whitelisted by extension and take them too). It never performs OCR:
+  a scanned or image-only PDF is *detected* and handed to the fallback
+  parser — the active OCR backend when OCR is on (see below), Docling when
+  it is installed, the legacy text parsers otherwise — and if nothing can
+  read the file the upload fails with a clear error instead of storing an
+  empty document.
+- `docling`: the previous default. Docling's layout and table models produce
+  structured Markdown, support OCR, and back the `read_document` tool's
+  `structured` output, at the cost of a large dependency tree (torch,
+  transformers, ONNX models) and seconds to minutes per PDF. Switching back is
+  this one variable; nothing else changes.
+
+With `DOC_PARSER_ENGINE=anydoc`, Docling still handles what anydoc cannot when
+it is installed: the fallback for files anydoc rejects, `.adoc`/`.vtt`/`.xml`
+chat attachments (those suffixes are not in the source-upload whitelist),
+and — when it is the OCR backend — scanned PDFs and images. Without Docling
+those formats use the standard parsers; OCR still works through the native
+backend, and with OCR off images are only read when `PARSE_IMAGE_REMOTE=true`.
+
+## Installing tesseract
+
+Nothing OCR-related ships in the base install: the default engine,
+`tesseract`, is a ~35 MB system package that you opt into like any other OCR
+dependency. Locally:
+
+```bash
+# Debian/Ubuntu                      # macOS
+apt-get install tesseract-ocr tesseract-ocr-eng    brew install tesseract
+```
+
+Docker images build without it by default; opt in with the build argument:
+
+```bash
+docker build --build-arg INSTALL_TESSERACT=true ./application
+```
+
+`deployment/docker-compose.yaml` forwards the same switch, so setting
+`INSTALL_TESSERACT=true` in `.env` (or the shell) bakes tesseract plus the
+English pack into locally built backend and worker images; `setup.sh` writes
+it when you answer yes to the OCR question after choosing to build images
+locally. Pre-built Docker Hub images (`docker-compose-hub.yaml`) do not
+include it, so `setup.sh` leaves OCR at its default (off) for them; to OCR
+there, point `OCR_ENGINE=deepseek` at a DeepSeek-OCR endpoint or install
+`tesseract-ocr` in a derived image. With `OCR_ENABLED=true` and no binary on
+`PATH`, scanned pages fail with an install hint (text-layer documents are
+unaffected).
+
+<Callout type="warning" emoji="⚠️">
+**Upgrading:** earlier images always included docling, and OCR ran through the
+RapidOCR engine bundled with it, so no system package was needed. The default
+image now ships neither docling nor tesseract. If your deployment uses the
+`tesseract` engine (the default) with `OCR_ENABLED=true` or
+`OCR_ATTACHMENTS_ENABLED=true`, add `INSTALL_TESSERACT=true` to `.env` before
+rebuilding, or OCR of scanned pages stops working after the rebuild.
+</Callout>
+
+## Installing the docling engine
+
+docling is not part of the base install, and OCR does not need it (see
+[OCR backends](#ocr-backends)). Add it when you want its layout-model OCR,
+`.adoc`/`.vtt`/`.xml` attachment parsing, or `read_document`'s `structured`
+output:
+
+```bash
+pip install -r application/requirements-docling.txt
+```
+
+Docker images build without it by default; opt in with the build argument:
+
+```bash
+docker build --build-arg INSTALL_DOCLING=true ./application
+```
+
+`deployment/docker-compose.yaml` forwards the same switch, so setting
+`INSTALL_DOCLING=true` in `.env` (or the shell) bakes docling into locally
+built backend and worker images; `setup.sh` offers it as a follow-up to the
+OCR question. Compose reads build arguments from the shell or from the
+`.env` you pass with `--env-file .env` (not from the containers' `env_file`),
+so build with `docker compose --env-file .env -f deployment/docker-compose.yaml build`
+as `setup.sh` does. Pre-built Docker Hub images (`docker-compose-hub.yaml`)
+never include docling; install it in a derived image instead. Either way no
+code changes are needed — docling is picked up as
+the fallback engine (and, under `OCR_BACKEND=auto`, as the OCR backend) as
+soon as it is importable, and `DOC_PARSER_ENGINE=docling` makes it the
+primary parser.
+
+## OCR
+
+OCR is optional and controlled by two on/off settings, a backend and an
+engine choice:
+
+```env
+OCR_ENABLED=false
+OCR_ATTACHMENTS_ENABLED=false
+OCR_BACKEND=auto
+OCR_ENGINE=tesseract
+```
+
+- `OCR_ENABLED`: OCR behavior for Source Docs ingestion.
+- `OCR_ATTACHMENTS_ENABLED`: OCR behavior for chat attachments uploaded from the message box.
+- `OCR_BACKEND`: which stack performs the OCR (next section).
+- `OCR_ENGINE`: which recognition engine it uses ([Choosing the OCR engine](#choosing-the-ocr-engine)).
+
+The older names `DOCLING_OCR_ENABLED` and `DOCLING_OCR_ATTACHMENTS_ENABLED`
+are still accepted as aliases.
+
+Under the default anydoc engine, a scanned PDF reaches OCR through anydoc's
+own detection: anydoc refuses it ("OCR is required") and the OCR backend
+takes over as the fallback parser. If that fallback also extracts almost
+nothing — OCR off, or no engine available — the upload fails with a clear
+message instead of silently indexing an empty document.
+
+Mixed documents — text pages with scanned pages among them — convert
+through anydoc, which reads the text pages and skips the scanned ones. With
+OCR on, DocsGPT probes every page's text layer, OCRs the pages that have
+none through the active backend, and appends their text; the document's
+metadata records the count as `ocr_pages`. With OCR off, only the text pages
+are indexed.
+
+## OCR backends
+
+Docling is not the only way to OCR. The `native` backend renders the pages
+that need it with pypdfium2 and Pillow — both already core dependencies —
+and feeds them straight into tesseract or a DeepSeek-OCR endpoint. No ML
+models load in the worker, and nothing beyond the ~35 MB tesseract binary
+(see [Installing tesseract](#installing-tesseract)) is needed.
+
+| `OCR_BACKEND` | What runs | When to pick it |
+|---|---|---|
+| `auto` (default) | Docling when the docling extra is installed, `native` otherwise. | Leave it: a plain install gets working OCR from tesseract alone, and installing docling upgrades OCR without touching config. |
+| `native` | pypdfium2 + Pillow page rendering into `tesseract` or `deepseek`. Pages that carry a text layer are read directly and never OCR'd; pages without one are rendered at `OCR_RENDER_DPI` (200) and OCR'd. Multi-frame TIFFs are read frame by frame. | You do not want docling's dependency tree or memory footprint, or you have docling installed for `structured` output but want lightweight OCR. |
+| `docling` | Docling's layout-model pipeline: hybrid OCR (only the bitmap regions of a page), reading-order recovery, TableFormer table structure, and the `auto` / `ocrmac` / `rapidocr` engines. | Multi-column scans, scanned tables you need as Markdown tables under tesseract, or macOS `ocrmac`. Needs the docling extra. |
+
+The trade-off is the layout model. Under `native`, multi-column scans rely
+on tesseract's own page segmentation and tesseract yields tables as plain
+lines; DeepSeek-OCR emits Markdown tables itself, so it remains the quality
+path on either backend. Under `DOC_PARSER_ENGINE=docling` with the `native`
+backend, PDFs whose every page has a text layer still go through Docling
+(OCR off) for its structured Markdown; only documents with scanned pages
+take the native path.
+
+## Choosing the OCR engine
+
+Benchmarked 2026-08 on English, bilingual EN/ZH, table-heavy and degraded
+scans (all engines driven through docling so layout handling is identical):
+
+| `OCR_ENGINE` | Backends | Role | Notes |
+|---|---|---|---|
+| `tesseract` | native, docling | **recommended default** | Best classic-engine accuracy in the bench: perfect EN word recall on all docs, 0.000 CER on the bilingual page, 100% table cells, robust to mild degradation. ~35 MB of *system* packages, CPU-only. Needs the `tesseract` binary + language packs — an optional install like every OCR dependency (see [Installing tesseract](#installing-tesseract)); set languages via `OCR_LANGS` (e.g. `eng+chi_sim`). |
+| `deepseek` | native, docling | best quality, heavy on the *server* | DeepSeek-OCR against an OpenAI-compatible endpoint. Only engine that reconstructs totals rows as table rows; near-perfect CJK; barely affected by degradation. The ingestion worker stays light (no layout models); the model runs in Ollama or vLLM. Costs: a GPU/Apple-Silicon endpoint, ~seconds per page, and occasional silent drops of page-level elements (titles). |
+| `auto` | docling | convenience | docling picks: `ocrmac` on macOS (excellent, ~1 s/page), `rapidocr` on Linux — see below before relying on it server-side. Also docling's automatic fallback whenever the selected engine is not installed. The native backend runs tesseract for it. |
+| `ocrmac` | docling | macOS only | Best raw accuracy and fastest of all classic engines; irrelevant for Linux deploys. |
+| `rapidocr` | docling | pip-only fallback | No system packages needed, perfect on tables/CJK — but it silently shreds some long text lines into garbage at every setting tried, which is content loss for RAG ingestion. Avoid as a server default until fixed upstream. |
+
+For `deepseek`, point the worker at an OpenAI-compatible endpoint:
+
+```env
+OCR_ENGINE=deepseek
+OCR_DEEPSEEK_URL=http://localhost:11434/v1/chat/completions  # Ollama default
+OCR_DEEPSEEK_MODEL=deepseek-ocr:3b
+OCR_DEEPSEEK_TIMEOUT=300  # seconds per page request, both backends
+```
+
+Ollama works out of the box (`ollama pull deepseek-ocr:3b`); for real
+throughput serve `deepseek-ai/DeepSeek-OCR` with vLLM on a GPU and set the
+URL accordingly. The native backend sends pages one at a time, so a slow
+laptop-hosted model only needs a generous `OCR_DEEPSEEK_TIMEOUT`; docling's
+VLM pipeline honours the same timeout per request but keeps its own four
+concurrent requests.
+
+A selected engine that is not available degrades rather than failing parses:
+under docling, a missing tesseract binary or non-macOS `ocrmac` falls back to
+`auto` with a warning; under `native`, a docling-only engine becomes
+tesseract, and a missing tesseract binary fails the scanned file with an
+install hint (text-layer documents are unaffected).
+
+<Callout type="warning" emoji="⚠️">
+A language listed in `OCR_LANGS` whose tesseract pack is not installed fails
+every scanned page loudly (tesseract exits with "Error opening data file
+... chi_sim.traineddata"), on both backends. Install the pack before
+listing it: `apt-get install tesseract-ocr-chi-sim` in the image, or on
+macOS download `chi_sim.traineddata` into `$(brew --prefix)/share/tessdata`.
+</Callout>
+
+### DeepSeek through docling versus native
+
+`OCR_ENGINE=deepseek` behaves differently on the two backends, and the
+difference matters more than for tesseract:
+
+- **native** sends one request per page that lacks a text layer and reads
+  every other page directly. Pages go one at a time with
+  `OCR_DEEPSEEK_TIMEOUT` per request, so a slow model server just takes
+  longer.
+- **docling** uses its VLM pipeline, which *replaces* the whole converter:
+  every page of every PDF that reaches docling goes to the model, text layer
+  or not (a 39-page text PDF measured 721 s against 20 s on the other paths),
+  with four concurrent requests and `OCR_DEEPSEEK_TIMEOUT` per request. In testing it also
+  dropped page-level elements — a title, an intro paragraph — that the same
+  model kept through the native path, and it logs a burst of pydantic
+  serialization warnings per conversion. Its one advantage was table
+  structure: it was the only configuration that returned an invoice's
+  subtotal/VAT/total rows as table rows.
+
+Measured on the same scans (2026-09, MacBook Air, Ollama `deepseek-ocr:3b`):
+
+| Configuration | Scanned page | Image | Accuracy | Tables | Chinese |
+|---|---|---|---|---|---|
+| tesseract, native | 1-2 s | 1-2 s | exact | flat lines | clean |
+| deepseek, native | 15-30 s | 15-25 s | exact | Markdown table, totals as bold lines | clean |
+| tesseract, docling | 3-4 s | ~8 s | exact | Markdown body, totals shredded | clean |
+| deepseek, docling | ~20 s | ~20 s | drops titles/paragraphs | exact | clean |
+
+Recommendation: `native` with `tesseract` for throughput, `native` with
+`deepseek` for quality, `docling` with `tesseract` when you want its layout
+model, and `docling` with `deepseek` only for table-heavy scans where the
+costs above are acceptable.
 
 ## Processing Flow
 
@@ -24,7 +248,7 @@ DOCLING_OCR_ATTACHMENTS_ENABLED=false
 1. Files are uploaded through `/api/upload`.
 2. Ingestion runs asynchronously in Celery (`ingest_worker`).
 3. `SimpleDirectoryReader` parses files with `get_default_file_extractor`.
-4. For PDFs and image formats, Docling parsers are used. OCR in this path is controlled by `DOCLING_OCR_ENABLED`.
+4. Documents are parsed by the `DOC_PARSER_ENGINE` engine; images (and, under `anydoc`, scanned PDFs) reach the OCR backend. OCR in this path is controlled by `OCR_ENABLED`.
 5. Parsed text is chunked, embedded, and stored in the vector store.
 6. Retrieval during chat uses this indexed text and returns source citations.
 
@@ -32,23 +256,30 @@ DOCLING_OCR_ATTACHMENTS_ENABLED=false
 
 1. Files are uploaded through `/api/store_attachment`.
 2. Celery task `attachment_worker` parses and stores the attachment in Postgres (`attachments` table).
-3. OCR in this path is controlled by `DOCLING_OCR_ATTACHMENTS_ENABLED`.
+3. OCR in this path is controlled by `OCR_ATTACHMENTS_ENABLED`.
 4. Attachments are not vectorized and are not added to the source index.
 5. During answer generation, selected attachment IDs are loaded and passed directly to the LLM pipeline.
 
 ## How Docling OCR Works
 
-Docling OCR behavior is different for PDFs vs images:
+With `OCR_BACKEND=docling`, OCR behavior is different for PDFs vs images:
 
 - PDF parser defaults to hybrid OCR:
   - text regions: extracted directly
   - bitmap/image regions: OCR only where needed
 - Image parser defaults to full-page OCR (the whole image is visual content).
 
-By default, Docling parser classes use RapidOCR options (language default: `english`).
+The engine and its languages come from `OCR_ENGINE` and `OCR_LANGS` (see the
+table above). `INSTALL_TESSERACT=true` installs only the English tesseract
+pack; for other languages install their packs in the image (e.g.
+`apt-get install tesseract-ocr-chi-sim`) and list them in `OCR_LANGS`
+(`eng+chi_sim`).
 
-<Callout type="info" emoji="ℹ️">
-Parser internals like OCR language and force-full-page OCR are currently set by code defaults, not separate `.env` settings.
+<Callout type="warning" emoji="⚠️">
+Upgrading from a RapidOCR-based deployment? RapidOCR covered English *and*
+Chinese with no configuration. The tesseract default only OCRs the languages
+in `OCR_LANGS` (`eng` out of the box), so CJK scans stop ingesting until
+their packs are installed and listed.
 </Callout>
 
 ### Model compilation
@@ -79,18 +310,19 @@ This means OCR quality is especially important for text fallback paths and for m
 
 ## Recommended Configuration
 
-For most OCR-enabled use cases, enable both flags:
+For most OCR-enabled use cases, enable both flags and leave the backend on
+`auto`:
 
 ```env
-DOCLING_OCR_ENABLED=true
-DOCLING_OCR_ATTACHMENTS_ENABLED=true
+OCR_ENABLED=true
+OCR_ATTACHMENTS_ENABLED=true
 ```
 
 After changing these settings, restart the API and Celery worker.
 
 ## Legacy Fallback Notes
 
-- If Docling is unavailable, DocsGPT falls back to legacy parsers.
+- If Docling is unavailable, DocsGPT falls back to the native OCR parsers (OCR on) or the legacy parsers (OCR off) for the formats anydoc does not cover, and anydoc's refusals of scanned PDFs become upload errors only when no OCR is available.
 - With OCR disabled, text-based PDFs can still parse, but scanned/image-heavy content may produce little text.
-- For image parsing without Docling OCR, the legacy image parser only extracts text when `PARSE_IMAGE_REMOTE=true`.
+- For image parsing without OCR, the legacy image parser only extracts text when `PARSE_IMAGE_REMOTE=true`.
 

frontend/src/constants/fileUpload.ts

@@ -26,6 +26,23 @@ export const FILE_UPLOAD_ACCEPT: Record<string, string[]> = {
   'application/vnd.openxmlformats-officedocument.presentationml.presentation': [
     '.pptx',
   ],
+  'application/xhtml+xml': ['.xhtml'],
+  'application/msword': ['.doc'],
+  'application/vnd.ms-word.document.macroEnabled.12': ['.docm'],
+  'application/vnd.oasis.opendocument.text': ['.odt'],
+  'application/rtf': ['.rtf'],
+  'text/rtf': ['.rtf'],
+  'application/vnd.ms-powerpoint': ['.ppt', '.pps', '.pot'],
+  'application/vnd.ms-powerpoint.presentation.macroEnabled.12': ['.pptm'],
+  'application/vnd.openxmlformats-officedocument.presentationml.slideshow': [
+    '.ppsx',
+  ],
+  'application/vnd.ms-powerpoint.slideshow.macroEnabled.12': ['.ppsm'],
+  'application/vnd.oasis.opendocument.presentation': ['.odp'],
+  'application/vnd.ms-excel': ['.xls'],
+  'application/vnd.ms-excel.sheet.macroEnabled.12': ['.xlsm'],
+  'application/vnd.ms-excel.sheet.binary.macroEnabled.12': ['.xlsb'],
+  'application/vnd.oasis.opendocument.spreadsheet': ['.ods'],
   'image/png': ['.png'],
   'image/jpeg': ['.jpeg'],
   'image/jpg': ['.jpg'],
@@ -45,6 +62,22 @@ export const FILE_UPLOAD_ACCEPT_ATTR = [
   '.epub',
   '.xlsx',
   '.pptx',
+  '.xhtml',
+  '.doc',
+  '.docm',
+  '.odt',
+  '.rtf',
+  '.ppt',
+  '.pptm',
+  '.pps',
+  '.ppsx',
+  '.ppsm',
+  '.pot',
+  '.odp',
+  '.xls',
+  '.xlsm',
+  '.xlsb',
+  '.ods',
   '.png',
   '.jpeg',
   '.jpg',
@@ -76,6 +109,22 @@ export const SOURCE_FILE_TREE_ACCEPT_ATTR = [
   '.json',
   '.xlsx',
   '.pptx',
+  '.xhtml',
+  '.doc',
+  '.docm',
+  '.odt',
+  '.rtf',
+  '.ppt',
+  '.pptm',
+  '.pps',
+  '.ppsx',
+  '.ppsm',
+  '.pot',
+  '.odp',
+  '.xls',
+  '.xlsm',
+  '.xlsb',
+  '.ods',
   '.png',
   '.jpg',
   '.jpeg',
@@ -105,6 +154,23 @@ export const ATTACHMENT_PARSER_EXTENSIONS: readonly string[] = [
   '.json',
   '.xlsx',
   '.pptx',
+  // Read by the anydoc engine, a core backend dependency: legacy and
+  // macro-enabled Office, OpenDocument, RTF.
+  '.docm',
+  '.doc',
+  '.odt',
+  '.rtf',
+  '.pptm',
+  '.ppt',
+  '.pps',
+  '.ppsx',
+  '.ppsm',
+  '.pot',
+  '.odp',
+  '.xlsm',
+  '.xlsb',
+  '.xls',
+  '.ods',
   '.adoc',
   '.asciidoc',
   '.png',

frontend/src/locale/de.json

@@ -765,7 +765,7 @@
       "start": "Chat starten",
       "name": "Name",
       "choose": "Dateien auswählen",
-      "info": "Bitte lade .pdf, .txt, .rst, .csv, .xlsx, .docx, .md, .html, .epub, .json, .pptx, .zip hoch (max. 25 MB)",
+      "info": "Bitte lade .pdf, .txt, .rst, .csv, .xlsx, .xlsm, .xlsb, .xls, .ods, .docx, .docm, .doc, .odt, .rtf, .md, .html, .xhtml, .png, .jpg, .jpeg, .epub, .json, .pptx, .pptm, .ppt, .pps, .ppsx, .ppsm, .pot, .odp, .zip hoch (max. 25 MB)",
       "uploadedFiles": "Hochgeladene Dateien",
       "cancel": "Abbrechen",
       "train": "Trainieren",

frontend/src/locale/en.json

@@ -770,7 +770,7 @@
       "start": "Start Chatting",
       "name": "Name",
       "choose": "Choose Files",
-      "info": "Please upload .pdf, .txt, .rst, .csv, .xlsx, .docx, .md, .html, .epub, .json, .pptx, .zip limited to 25mb",
+      "info": "Please upload .pdf, .txt, .rst, .csv, .xlsx, .xlsm, .xlsb, .xls, .ods, .docx, .docm, .doc, .odt, .rtf, .md, .html, .xhtml, .png, .jpg, .jpeg, .epub, .json, .pptx, .pptm, .ppt, .pps, .ppsx, .ppsm, .pot, .odp, .zip limited to 25mb",
       "uploadedFiles": "Uploaded Files",
       "cancel": "Cancel",
       "train": "Train",

frontend/src/locale/es.json

@@ -765,7 +765,7 @@
       "start": "Comenzar a chatear",
       "name": "Nombre",
       "choose": "Seleccionar Archivos",
-      "info": "Por favor, sube archivos .pdf, .txt, .rst, .csv, .xlsx, .docx, .md, .html, .epub, .json, .pptx, .zip limitados a 25MB",
+      "info": "Por favor, sube archivos .pdf, .txt, .rst, .csv, .xlsx, .xlsm, .xlsb, .xls, .ods, .docx, .docm, .doc, .odt, .rtf, .md, .html, .xhtml, .png, .jpg, .jpeg, .epub, .json, .pptx, .pptm, .ppt, .pps, .ppsx, .ppsm, .pot, .odp, .zip limitados a 25MB",
       "uploadedFiles": "Archivos Subidos",
       "cancel": "Cancelar",
       "train": "Entrenar",

frontend/src/locale/jp.json

@@ -765,7 +765,7 @@
       "start": "チャットを開始する",
       "name": "名前",
       "choose": "ファイルを選択",
-      "info": "25MBまでの.pdf、.txt、.rst、.csv、.xlsx、.docx、.md、.html、.epub、.json、.pptx、.zipファイルをアップロードしてください",
+      "info": "25MBまでの.pdf、.txt、.rst、.csv、.xlsx、.xlsm、.xlsb、.xls、.ods、.docx、.docm、.doc、.odt、.rtf、.md、.html、.xhtml、.png、.jpg、.jpeg、.epub、.json、.pptx、.pptm、.ppt、.pps、.ppsx、.ppsm、.pot、.odp、.zipファイルをアップロードしてください",
       "uploadedFiles": "アップロードされたファイル",
       "cancel": "キャンセル",
       "train": "トレーニング",

frontend/src/locale/ru.json

@@ -765,7 +765,7 @@
       "start": "Начать чат",
       "name": "Имя",
       "choose": "Выбрать файлы",
-      "info": "Пожалуйста, загрузите файлы .pdf, .txt, .rst, .csv, .xlsx, .docx, .md, .html, .epub, .json, .pptx, .zip размером до 25 МБ",
+      "info": "Пожалуйста, загрузите файлы .pdf, .txt, .rst, .csv, .xlsx, .xlsm, .xlsb, .xls, .ods, .docx, .docm, .doc, .odt, .rtf, .md, .html, .xhtml, .png, .jpg, .jpeg, .epub, .json, .pptx, .pptm, .ppt, .pps, .ppsx, .ppsm, .pot, .odp, .zip размером до 25 МБ",
       "uploadedFiles": "Загруженные файлы",
       "cancel": "Отмена",
       "train": "Тренировка",

frontend/src/locale/zh-TW.json

@@ -765,7 +765,7 @@
       "start": "開始對話",
       "name": "名稱",
       "choose": "選擇檔案",
-      "info": "請上傳限制為25MB的.pdf、.txt、.rst、.csv、.xlsx、.docx、.md、.html、.epub、.json、.pptx、.zip檔案",
+      "info": "請上傳限制為25MB的.pdf、.txt、.rst、.csv、.xlsx、.xlsm、.xlsb、.xls、.ods、.docx、.docm、.doc、.odt、.rtf、.md、.html、.xhtml、.png、.jpg、.jpeg、.epub、.json、.pptx、.pptm、.ppt、.pps、.ppsx、.ppsm、.pot、.odp、.zip檔案",
       "uploadedFiles": "已上傳檔案",
       "cancel": "取消",
       "train": "訓練",

frontend/src/locale/zh.json

@@ -765,7 +765,7 @@
       "start": "开始聊天",
       "name": "名称",
       "choose": "选择文件",
-      "info": "请上传限制为25MB的.pdf、.txt、.rst、.csv、.xlsx、.docx、.md、.html、.epub、.json、.pptx、.zip文件",
+      "info": "请上传限制为25MB的.pdf、.txt、.rst、.csv、.xlsx、.xlsm、.xlsb、.xls、.ods、.docx、.docm、.doc、.odt、.rtf、.md、.html、.xhtml、.png、.jpg、.jpeg、.epub、.json、.pptx、.pptm、.ppt、.pps、.ppsx、.ppsm、.pot、.odp、.zip文件",
       "uploadedFiles": "已上传文件",
       "cancel": "取消",
       "train": "训练",

setup.ps1

@@ -513,10 +513,29 @@ function Configure-DocProcessing {
         Write-ColorText "PDF-as-image parsing enabled." -ForegroundColor "Green"
     }
 
-    $ocr_enabled = Read-Host "Enable OCR for document processing (Docling)? (y/N)"
+    # OCR needs the tesseract binary, an optional system package that only
+    # locally built images can include (INSTALL_TESSERACT build arg). The
+    # pre-built Docker Hub images ship without it, so there OCR stays off
+    # (its default) rather than being switched on to fail on every scan.
+    if ($COMPOSE_FILE -ne $COMPOSE_FILE_LOCAL) {
+        Write-ColorText "OCR for scanned PDFs and images stays off: the pre-built Docker Hub images do not include tesseract. To use OCR, rerun setup and choose option 5 (build images locally), or use a DeepSeek-OCR endpoint by adding OCR_ENABLED=true, OCR_ENGINE=deepseek and OCR_DEEPSEEK_URL=<endpoint> to .env." -ForegroundColor "Yellow"
+        return
+    }
+
+    $ocr_enabled = Read-Host "Enable OCR for scanned PDFs and images? (y/N)"
     if ($ocr_enabled -eq "y" -or $ocr_enabled -eq "Y") {
-        "DOCLING_OCR_ENABLED=true" | Add-Content -Path $ENV_FILE -Encoding utf8
-        Write-ColorText "Docling OCR enabled." -ForegroundColor "Green"
+        "OCR_ENABLED=true" | Add-Content -Path $ENV_FILE -Encoding utf8
+        # Bakes tesseract into the locally built images (docker compose
+        # --env-file .env build).
+        "INSTALL_TESSERACT=true" | Add-Content -Path $ENV_FILE -Encoding utf8
+        Write-ColorText "OCR enabled. tesseract will be built into the images (INSTALL_TESSERACT=true); for a DeepSeek-OCR endpoint instead, set OCR_ENGINE=deepseek and OCR_DEEPSEEK_URL=<endpoint> in .env." -ForegroundColor "Green"
+        $docling_ocr = Read-Host "Also install the Docling layout engine for OCR (better tables/reading order, several GB heavier)? (y/N)"
+        if ($docling_ocr -eq "y" -or $docling_ocr -eq "Y") {
+            # Locally built images include docling via this build arg; it becomes
+            # the OCR backend automatically (OCR_BACKEND=auto).
+            "INSTALL_DOCLING=true" | Add-Content -Path $ENV_FILE -Encoding utf8
+            Write-ColorText "Docling will be built into locally built images (docker compose --env-file .env build). Pre-built Docker Hub images do not include it." -ForegroundColor "Green"
+        }
     }
 }
 
@@ -633,7 +652,14 @@ function Use-DocsPublicAPIEndpoint {
             throw "Docker compose pull failed with exit code $LASTEXITCODE"
         }
         
-        & docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" up -d
+        # Locally built images must be rebuilt rather than reused:
+        # INSTALL_TESSERACT and INSTALL_DOCLING are build args, so a rerun that
+        # switches OCR on would otherwise keep the image built without them.
+        $up_args = @("up", "-d")
+        if ($COMPOSE_FILE -eq $COMPOSE_FILE_LOCAL) {
+            $up_args = @("up", "--build", "-d")
+        }
+        & docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" @up_args
         if ($LASTEXITCODE -ne 0) {
             throw "Docker compose up failed with exit code $LASTEXITCODE"
         }

setup.sh

@@ -367,10 +367,29 @@ configure_doc_processing() {
         echo -e "${GREEN}PDF-as-image parsing enabled.${NC}"
     fi
 
-    read -p "$(echo -e "${DEFAULT_FG}Enable OCR for document processing (Docling)? (y/N): ${NC}")" ocr_enabled
+    # OCR needs the tesseract binary, an optional system package that only
+    # locally built images can include (INSTALL_TESSERACT build arg). The
+    # pre-built Docker Hub images ship without it, so there OCR stays off
+    # (its default) rather than being switched on to fail on every scan.
+    if [[ "$COMPOSE_FILE" != "$COMPOSE_FILE_LOCAL" ]]; then
+        echo -e "${YELLOW}OCR for scanned PDFs and images stays off: the pre-built Docker Hub images do not include tesseract. To use OCR, rerun setup and choose option 5 (build images locally), or use a DeepSeek-OCR endpoint by adding OCR_ENABLED=true, OCR_ENGINE=deepseek and OCR_DEEPSEEK_URL=<endpoint> to .env.${NC}"
+        return
+    fi
+
+    read -p "$(echo -e "${DEFAULT_FG}Enable OCR for scanned PDFs and images? (y/N): ${NC}")" ocr_enabled
     if [[ "$ocr_enabled" =~ ^[yY]$ ]]; then
-        echo "DOCLING_OCR_ENABLED=true" >> "$ENV_FILE"
-        echo -e "${GREEN}Docling OCR enabled.${NC}"
+        echo "OCR_ENABLED=true" >> "$ENV_FILE"
+        # Bakes tesseract into the locally built images (docker compose
+        # --env-file .env build).
+        echo "INSTALL_TESSERACT=true" >> "$ENV_FILE"
+        echo -e "${GREEN}OCR enabled. tesseract will be built into the images (INSTALL_TESSERACT=true); for a DeepSeek-OCR endpoint instead, set OCR_ENGINE=deepseek and OCR_DEEPSEEK_URL=<endpoint> in .env.${NC}"
+        read -p "$(echo -e "${DEFAULT_FG}Also install the Docling layout engine for OCR (better tables/reading order, several GB heavier)? (y/N): ${NC}")" docling_ocr
+        if [[ "$docling_ocr" =~ ^[yY]$ ]]; then
+            # Locally built images include docling via this build arg; it becomes
+            # the OCR backend automatically (OCR_BACKEND=auto).
+            echo "INSTALL_DOCLING=true" >> "$ENV_FILE"
+            echo -e "${GREEN}Docling will be built into locally built images (docker compose --env-file .env build). Pre-built Docker Hub images do not include it.${NC}"
+        fi
     fi
 }
 
@@ -455,7 +474,14 @@ use_docs_public_api_endpoint() {
     check_and_start_docker
 
     echo -e "\n${NC}Starting Docker Compose...${NC}"
-    docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" pull && docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" up -d
+    # Locally built images must be rebuilt rather than reused: INSTALL_TESSERACT
+    # and INSTALL_DOCLING are build args, so a rerun that switches OCR on would
+    # otherwise keep the image that was built without them.
+    up_args=(up -d)
+    if [ "$COMPOSE_FILE" = "$COMPOSE_FILE_LOCAL" ]; then
+        up_args=(up --build -d)
+    fi
+    docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" pull && docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" "${up_args[@]}"
     docker_compose_status=$? # Capture exit status of docker compose
 
     echo "Docker Compose Exit Status: $docker_compose_status"

tests/parser/file/fixtures/nda_en_zh_cid_font.pdf

@@ -0,0 +1,101 @@
+%PDF-1.4
+%“Œ‹ž ReportLab Generated PDF document (opensource)
+1 0 obj
+<<
+/F1 2 0 R /F2 3 0 R /F3 4 0 R /F4 5 0 R
+>>
+endobj
+2 0 obj
+<<
+/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
+>>
+endobj
+3 0 obj
+<<
+/BaseFont /Times-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
+>>
+endobj
+4 0 obj
+<<
+/BaseFont /STSong-Light /DescendantFonts [ <<
+/BaseFont /STSong-Light /CIDSystemInfo <<
+/Ordering (GB1) /Registry (Adobe) /Supplement 0
+>> /DW 1000 /FontDescriptor <<
+/Ascent 752 /CapHeight 737 /Descent -271 /Flags 6 /FontBBox [ -25 -254 1000 880 ] /FontName /STSongStd-Light 
+  /ItalicAngle 0 /Leading 148 /MaxWidth 1000 /MissingWidth 500 /StemH 91 /StemV 58 
+  /Type /FontDescriptor /XHeight 553
+>> /Subtype /CIDFontType0 /Type /Font 
+  /W [ 1 [ 207 270 342 467 462 797 710 239 374 ] 10 [ 374 423 605 238 375 238 334 462 ] 18 26 462 27 28 238 
+  29 31 605 32 [ 344 748 684 560 695 739 563 511 729 793 
+  318 312 666 526 896 758 772 544 772 628 
+  465 607 753 711 972 647 620 607 374 333 
+  374 606 500 239 417 503 427 529 415 264 
+  444 518 241 230 495 228 793 527 524 ] 81 [ 524 504 338 336 277 517 450 652 466 452 
+  407 370 258 370 605 ] ]
+>> ] /Encoding /UniGB-UCS2-H /Name /F3 /Subtype /Type0 /Type /Font
+>>
+endobj
+5 0 obj
+<<
+/BaseFont /Times-Roman /Encoding /WinAnsiEncoding /Name /F4 /Subtype /Type1 /Type /Font
+>>
+endobj
+6 0 obj
+<<
+/Contents 10 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 9 0 R /Resources <<
+/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
+>> /Rotate 0 /Trans <<
+
+>> 
+  /Type /Page
+>>
+endobj
+7 0 obj
+<<
+/PageMode /UseNone /Pages 9 0 R /Type /Catalog
+>>
+endobj
+8 0 obj
+<<
+/Author (\(anonymous\)) /CreationDate (D:20260822115950+02'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260822115950+02'00') /Producer (ReportLab PDF Library - \(opensource\)) 
+  /Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
+>>
+endobj
+9 0 obj
+<<
+/Count 1 /Kids [ 6 0 R ] /Type /Pages
+>>
+endobj
+10 0 obj
+<<
+/Filter [ /ASCII85Decode /FlateDecode ] /Length 1647
+>>
+stream
+Gatm<gN):C&:O:SoKt6S$HmLQ&3Rqa-FV*"%Nm-aMW-4=#+,j_=@nmi2Z)t-)*e6=$[Ld,&=Q79:X>PPVJQYC5?'k&+i&d>B>799%,;FkdkJW*5heHuF@91dXh%AJZ6fb3i'sKhCnMdY'0VkEO4GA.p;"DJY62!"YAIse&.WT60%ek]="]O%+5Ot<#Fl%^o]il7fm2dGp=dm)m^:gDD9"d?pER14WDHmu']GE^2k$Vs`2KagWk")2f26\M]Bs+K@uNQtga3OQkLQ7H-hY4>1!4.4XoK+nl.ZlK#$d"IN+=9q.'8T'8cF3g*[-3:LS5c>fi!l0mSVa4QpjYgA<$0LC`2-c:@^;pCa)ekN>!I%&S3he7oPPIfI^JJ8"'-"K,>QL6DaesnS'ThOk/)63Q<B9b;^E^!3N(^L"g8T-_[_e%ED-JKcd/L:T#=nOk`'\l#7Q=,(RK;`0#S<T4BU:`HhPGE"*QrTjD+LciNhe\[arIY[:*6r!Z(b46[T;c]'eT2ib4:#PYiS4KbfJkaWh^D3%L,2kV2tNtL-eVVV!V=WcZC*LmZ1JF,Q`_WpI/i"9jUr1Gd(ISB[Id?[]g@-RLQD"d?"iI[81]"&B'nLTX>D<J'G5>]1d]+<&nMsjUuVm%-`_Dn^N7Y3@(%aO`>?OaQR7"S`I)*4n^gDS'@E((C?3'qDA#_!0S=@#*`D<h%J/PD"f$4CY;-(VXO;.Ngc^I+,-r\]qpUQHC*nl`bn!js*R3SkJqAMhH<a9;!@-5%1$9AH:,n;b&'!\_DddM7mc;^=s`,BOb&%SnDA7Rl;HLN*t%(`jGJS&".8T-M[a.`>blJkd8M[QOjG@Lffk@WSsIihjLDN`q?@n!YaFRc6etUSZ3TeZg#m?mL-MTMO5V)b]\Em&*E2^jCc`Xsk1<"JLgnj,EWu^Kg\"3.u<u5YCaES/ZDf5DpeMd24qDipat:7Oab*_jqT_s6MJ(i_J60;p<ik,/MGX2+u90)Xt!desS2A+)qeT6sUK/IJ'C=1<0u0>SaqZ<FhNoMfaf*!B2`NiLP;eB4?[`;/Y%o&Gm)'qO1JK0uckb!lh3>=t73#j/BK3<dLaZd96%gq]>W+8,V.,0D<(=M`#OG+,1-Fh</5[IM)#q0>k-!*=d[eI7&!b)UOUK$3XO.Y5HWgM[D<SAa`SCR9MN]H2Q[$5(+ZDUq;e-9V7T89Qo[Rm`4fG0o4@V.p48_A)/:trd:.`nh*S%-%oo[?Go6OFLjPL]dfcTW*/@I<%u%sqi96p7;=;:bYI+:AU5':P>$c<"Wh_I_.r/0:j+[",VmP^3"R7]Vuj1Bju_M4fQpgg+K1-[?e$i#W^p3[3m^DdEM$S$d\G^r%4.<YC,K9<kt81fnor-c7+%aA'BW<5M_bT$PN_@IR!Yssf=Ie",juu4&#3f]0+Uq".M]_sN3S\fN4OR!lJL2(a=:b4Ff&7E;q)O_P#WqT?;^@'e:m]2q4Y\']#rYS`0J,.E<b/Dai'5?mlc6AhJna\JGg_:Yc:98:nsRLal!#c+EsM(<tb(J^cTEVjq!Ll`V)/=\n1?2VP!nUibVne=IuE;BNdBFoimp4D_?%=$%6*-QW3f0rnX&<H85'1fg=@QlClK`^N0$X\\#1DFp#6kjo8*>*l66RfIdu>~>endstream
+endobj
+xref
+0 11
+0000000000 65535 f 
+0000000061 00000 n 
+0000000122 00000 n 
+0000000229 00000 n 
+0000000337 00000 n 
+0000001270 00000 n 
+0000001379 00000 n 
+0000001583 00000 n 
+0000001651 00000 n 
+0000001931 00000 n 
+0000001990 00000 n 
+trailer
+<<
+/ID 
+[<6167f87b12d550221d4edd2d2c131d94><6167f87b12d550221d4edd2d2c131d94>]
+% ReportLab generated PDF document -- digest (opensource)
+
+/Info 8 0 R
+/Root 7 0 R
+/Size 11
+>>
+startxref
+3729
+%%EOF

tests/parser/file/test_anydoc_parser.py

@@ -0,0 +1,614 @@
+"""Tests for the anydoc parser (the default ``DOC_PARSER_ENGINE``).
+
+Three behaviours matter and are asserted separately:
+
+* a file anydoc converts comes back as Markdown, with ``last_engine`` set;
+* a file it refuses (a scanned PDF, an unknown format) goes to the fallback
+  parser when one is configured, and every failure surfaces as
+  ``DocumentParseError`` — never as an empty document or a raw traceback;
+* the suffix list the parser map is built from matches what the installed
+  anydoc actually accepts.
+"""
+
+import importlib.machinery
+import sys
+import types
+from pathlib import Path
+
+import pytest
+
+from application.parser.file.base_parser import BaseParser, DocumentParseError
+
+anydoc = pytest.importorskip("anydoc")
+
+from application.parser.file.anydoc_parser import (  # noqa: E402 — after importorskip
+    ANYDOC_SUFFIXES,
+    AnydocParser,
+    anydoc_available,
+)
+
+
+# Long enough to clear the scanned-PDF near-empty guard (_MIN_SCAN_FALLBACK_CHARS).
+FALLBACK_TEXT = "fallback parser text output, long enough to clear the scanned-PDF guard."
+
+
+class _RecordingFallback(BaseParser):
+    """Stands in for docling / a legacy parser so delegation is observable."""
+
+    def __init__(self, result=FALLBACK_TEXT):
+        super().__init__(parser_config={})
+        self.calls = []
+        self._result = result
+
+    def _init_parser(self):
+        return {}
+
+    def parse_file(self, file, errors="ignore"):
+        self.calls.append(file)
+        return self._result
+
+
+class _ExplodingFallback(BaseParser):
+    def _init_parser(self):
+        return {}
+
+    def parse_file(self, file, errors="ignore"):
+        raise RuntimeError("fallback blew up")
+
+
+def _scanned_pdf(path, pages=2):
+    """A PDF with pages but no text layer: what a scan looks like to a parser."""
+    from pypdf import PdfWriter
+
+    writer = PdfWriter()
+    for _ in range(pages):
+        writer.add_blank_page(width=612, height=792)
+    with open(path, "wb") as fh:
+        writer.write(fh)
+    return path
+
+
+def _fake_anydoc(monkeypatch, to_markdown):
+    """Install a stub ``anydoc`` module whose ``to_markdown`` is ``to_markdown``."""
+    fake = types.ModuleType("anydoc")
+    fake.__spec__ = importlib.machinery.ModuleSpec("anydoc", None)
+
+    class ConvertError(Exception):
+        pass
+
+    class UnsupportedError(ConvertError):
+        pass
+
+    class ResourceLimitError(ConvertError):
+        pass
+
+    fake.ConvertError = ConvertError
+    fake.UnsupportedError = UnsupportedError
+    fake.ResourceLimitError = ResourceLimitError
+    fake.to_markdown = to_markdown
+    monkeypatch.setitem(sys.modules, "anydoc", fake)
+    return fake
+
+
+# --- the suffix list is what anydoc really supports ---------------------------
+
+
+def test_every_listed_suffix_is_an_anydoc_format():
+    for suffix in ANYDOC_SUFFIXES:
+        assert anydoc.format_from_extension(suffix) is not None, suffix
+
+
+def test_epub_and_html_are_not_routed_to_anydoc():
+    assert ".epub" not in ANYDOC_SUFFIXES
+    assert ".html" not in ANYDOC_SUFFIXES
+
+
+# --- successful conversion -----------------------------------------------------
+
+
+def test_csv_converts_to_markdown_table(tmp_path):
+    path = tmp_path / "items.csv"
+    path.write_text("name,qty\nwidget,3\ngadget,5\n")
+    parser = AnydocParser()
+    parser.init_parser()
+
+    out = parser.parse_file(path)
+
+    assert isinstance(out, str)
+    assert "widget" in out and "gadget" in out
+    assert "|" in out  # rendered as a GFM table, not flat text
+    assert parser.last_engine == "anydoc"
+
+
+def test_xlsx_converts_with_cell_text(tmp_path):
+    openpyxl = pytest.importorskip("openpyxl")
+    path = tmp_path / "book.xlsx"
+    wb = openpyxl.Workbook()
+    ws = wb.active
+    ws.append(["city", "population"])
+    ws.append(["Ljubljana", 295000])
+    wb.save(path)
+
+    out = AnydocParser().parse_file(path)
+
+    assert "Ljubljana" in out
+    assert "295000" in out
+
+
+def test_accepts_path_and_str(tmp_path):
+    path = tmp_path / "a.csv"
+    path.write_text("x,y\n1,2\n")
+    assert AnydocParser().parse_file(str(path)) == AnydocParser().parse_file(path)
+
+
+# --- refusal routes to the fallback --------------------------------------------
+
+
+def test_scanned_pdf_delegates_to_fallback(tmp_path):
+    path = _scanned_pdf(tmp_path / "scan.pdf")
+    fallback = _RecordingFallback()
+    parser = AnydocParser(fallback_parser=fallback)
+
+    out = parser.parse_file(path)
+
+    assert out == FALLBACK_TEXT
+    assert fallback.calls == [path]
+    assert parser.last_engine == "_RecordingFallback"
+
+
+def test_fallback_last_engine_is_reported_when_it_has_one(tmp_path):
+    path = _scanned_pdf(tmp_path / "scan.pdf")
+    fallback = _RecordingFallback()
+    fallback.last_engine = "pypdfium2"
+
+    parser = AnydocParser(fallback_parser=fallback)
+    parser.parse_file(path)
+
+    assert parser.last_engine == "pypdfium2"
+
+
+def test_scanned_pdf_without_fallback_is_document_parse_error(tmp_path):
+    path = _scanned_pdf(tmp_path / "scan.pdf")
+    parser = AnydocParser()
+
+    with pytest.raises(DocumentParseError, match="scan.pdf"):
+        parser.parse_file(path)
+    assert parser.last_engine is None
+
+
+def test_unknown_format_without_fallback_is_document_parse_error(tmp_path):
+    path = tmp_path / "blob.xyz"
+    path.write_bytes(b"hello")
+
+    with pytest.raises(DocumentParseError, match="blob.xyz"):
+        AnydocParser().parse_file(path)
+
+
+def test_fallback_failure_is_document_parse_error(tmp_path):
+    """A fallback that raises must not leak a bare exception (load_data skips only DocumentParseError)."""
+    path = _scanned_pdf(tmp_path / "scan.pdf")
+
+    with pytest.raises(DocumentParseError):
+        AnydocParser(fallback_parser=_ExplodingFallback()).parse_file(path)
+
+
+def test_fallback_document_parse_error_passes_through(tmp_path):
+    class _Refusing(BaseParser):
+        def _init_parser(self):
+            return {}
+
+        def parse_file(self, file, errors="ignore"):
+            raise DocumentParseError("scan needs OCR")
+
+    path = _scanned_pdf(tmp_path / "scan.pdf")
+    with pytest.raises(DocumentParseError, match="scan needs OCR"):
+        AnydocParser(fallback_parser=_Refusing()).parse_file(path)
+
+
+def test_missing_file_is_document_parse_error(tmp_path):
+    with pytest.raises(DocumentParseError, match="could not be read"):
+        AnydocParser().parse_file(tmp_path / "missing.docx")
+
+
+def test_empty_output_delegates(tmp_path, monkeypatch):
+    """Whitespace-only output is treated as a failed conversion, not stored."""
+    _fake_anydoc(monkeypatch, lambda path: "  \n\n ")
+    fallback = _RecordingFallback("real text")
+    path = tmp_path / "x.docx"
+    path.write_bytes(b"irrelevant")
+
+    out = AnydocParser(fallback_parser=fallback).parse_file(path)
+
+    assert out == "real text"
+    assert fallback.calls == [path]
+
+
+def test_typed_convert_error_delegates(tmp_path, monkeypatch):
+    fake = _fake_anydoc(monkeypatch, None)
+
+    def _refuse(path):
+        raise fake.UnsupportedError("PDF has no extractable text; OCR is required")
+
+    fake.to_markdown = _refuse
+    fallback = _RecordingFallback()
+    path = tmp_path / "x.pdf"
+    path.write_bytes(b"%PDF-1.4")
+
+    assert AnydocParser(fallback_parser=fallback).parse_file(path) == FALLBACK_TEXT
+
+
+def test_typed_convert_error_message_reaches_the_user(tmp_path, monkeypatch):
+    fake = _fake_anydoc(monkeypatch, None)
+
+    def _refuse(path):
+        raise fake.UnsupportedError("PDF has no extractable text; OCR is required")
+
+    fake.to_markdown = _refuse
+    path = tmp_path / "x.pdf"
+    path.write_bytes(b"%PDF-1.4")
+
+    with pytest.raises(DocumentParseError, match="OCR is required"):
+        AnydocParser().parse_file(path)
+
+
+def test_resource_limit_is_terminal_even_with_fallback(tmp_path, monkeypatch):
+    """anydoc refusing a file as too expensive must not hand it to a heavier
+    engine that would then spend exactly what was refused."""
+    fake = _fake_anydoc(monkeypatch, None)
+
+    def _refuse(path):
+        raise fake.ResourceLimitError(
+            "resource limit exceeded (max_entry_bytes): word/document.xml declares 400400167 bytes"
+        )
+
+    fake.to_markdown = _refuse
+    fallback = _RecordingFallback()
+    parser = AnydocParser(fallback_parser=fallback)
+    path = tmp_path / "bomb.docx"
+    path.write_bytes(b"PK")
+
+    with pytest.raises(DocumentParseError, match="max_entry_bytes"):
+        parser.parse_file(path)
+    assert fallback.calls == []
+    assert parser.last_engine is None
+
+
+def test_unexpected_exception_is_document_parse_error(tmp_path, monkeypatch):
+    def _crash(path):
+        raise RuntimeError("panic")
+
+    _fake_anydoc(monkeypatch, _crash)
+    path = tmp_path / "x.docx"
+    path.write_bytes(b"irrelevant")
+
+    with pytest.raises(DocumentParseError, match="x.docx"):
+        AnydocParser().parse_file(path)
+
+
+# --- init / availability --------------------------------------------------------
+
+
+def test_init_parser_records_fallback():
+    parser = AnydocParser(fallback_parser=_RecordingFallback())
+    parser.init_parser()
+    assert parser.parser_config == {"fallback_parser": "_RecordingFallback"}
+
+    bare = AnydocParser()
+    bare.init_parser()
+    assert bare.parser_config == {"fallback_parser": None}
+
+
+def test_init_parser_without_anydoc_raises_import_error(monkeypatch):
+    monkeypatch.setitem(sys.modules, "anydoc", None)
+    assert not anydoc_available()
+    with pytest.raises(ImportError, match="firecrawl-anydoc"):
+        AnydocParser().init_parser()
+
+
+def test_init_parser_imports_for_real_not_just_find_spec(monkeypatch):
+    """A wheel whose native extension fails to load has a spec but no importable
+    module; that must surface at init, not as a bare ImportError from parse_file
+    mid-ingest (which load_data does not catch)."""
+    from application.parser.file import anydoc_parser as mod
+
+    monkeypatch.setattr(mod, "anydoc_available", lambda: True)
+    monkeypatch.setitem(sys.modules, "anydoc", None)
+
+    with pytest.raises(ImportError, match="firecrawl-anydoc"):
+        AnydocParser().init_parser()
+
+
+# --- PDF trust check + tableize wiring (PR 3) -----------------------------------
+
+FIXTURES = Path(__file__).parent / "fixtures"
+CID_PDF = FIXTURES / "nda_en_zh_cid_font.pdf"
+
+
+# Long enough to clear the near-empty guard a trust-check re-parse must pass.
+_REROUTE_TEXT = "docling reroute text, long enough to clear the near-empty reroute guard"
+
+
+class _FakeDoclingFallback:
+    """Registered as a DoclingParser subclass so ``_is_docling_backed`` is True."""
+
+    def __new__(cls):
+        from application.parser.file.docling_parser import DoclingParser
+
+        class _Inner(DoclingParser):
+            def __init__(self):
+                super().__init__(ocr_enabled=False)
+                self._parser_config = {}
+                self.calls = []
+                self.last_engine = None
+
+            def parse_file(self, file, errors="ignore"):
+                self.calls.append(file)
+                return _REROUTE_TEXT
+
+        return _Inner()
+
+
+def test_trust_flagged_pdf_reroutes_to_docling_fallback():
+    fallback = _FakeDoclingFallback()
+    parser = AnydocParser(fallback_parser=fallback)
+
+    out = parser.parse_file(CID_PDF)
+
+    assert out == _REROUTE_TEXT
+    assert fallback.calls == [CID_PDF]
+    assert parser.last_engine == "_Inner"
+    assert parser.get_file_metadata(CID_PDF) == {}  # rerouted, nothing to warn about
+
+
+def test_trust_flagged_pdf_without_docling_keeps_output_and_warns():
+    parser = AnydocParser(fallback_parser=_RecordingFallback())  # not docling-backed
+
+    out = parser.parse_file(CID_PDF)
+
+    assert "NON-DISCLOSURE" in out.upper() or len(out) > 50  # anydoc's own output kept
+    meta = parser.get_file_metadata(CID_PDF)
+    assert "parse_warnings" in meta
+    assert any("ToUnicode" in w for w in meta["parse_warnings"])
+    assert any("CJK" in w for w in meta["parse_warnings"])
+    assert parser.last_engine == "anydoc"
+
+
+def test_trust_check_disabled_stamps_nothing(monkeypatch):
+    from application.parser.file import anydoc_parser as ap
+
+    monkeypatch.setattr(ap.settings, "PDF_TRUST_CHECK", False)
+    parser = AnydocParser()
+
+    parser.parse_file(CID_PDF)
+
+    assert parser.get_file_metadata(CID_PDF) == {}
+
+
+def test_trust_reroute_failure_keeps_anydoc_output():
+    fallback = _FakeDoclingFallback()
+
+    def _boom(file, errors="ignore"):
+        raise RuntimeError("docling exploded")
+
+    fallback.parse_file = _boom
+    parser = AnydocParser(fallback_parser=fallback)
+
+    out = parser.parse_file(CID_PDF)
+
+    assert "docling" not in out
+    assert "parse_warnings" in parser.get_file_metadata(CID_PDF)
+    assert parser.last_engine == "anydoc"
+
+
+def test_trust_reroute_near_empty_keeps_anydoc_output():
+    """A docling pipeline dropout ('' / '<!-- image -->') returns without
+    raising; adopting it would swap anydoc's real text for an empty document."""
+    fallback = _FakeDoclingFallback()
+    fallback.parse_file = lambda file, errors="ignore": "<!-- image -->"
+    parser = AnydocParser(fallback_parser=fallback)
+
+    out = parser.parse_file(CID_PDF)
+
+    assert "<!-- image -->" not in out
+    assert "parse_warnings" in parser.get_file_metadata(CID_PDF)
+    assert parser.last_engine == "anydoc"
+
+
+def test_warnings_reset_between_files(tmp_path):
+    parser = AnydocParser()
+    parser.parse_file(CID_PDF)
+    assert parser.get_file_metadata(CID_PDF) != {}
+
+    clean = tmp_path / "clean.csv"
+    clean.write_text("a,b\n1,2\n")
+    parser.parse_file(clean)
+
+    assert parser.get_file_metadata(clean) == {}
+    assert parser.get_file_metadata(CID_PDF) == {}
+
+
+def test_trust_check_errors_never_fail_the_parse(monkeypatch, tmp_path):
+    def _explode(path, markdown):
+        raise RuntimeError("scanner bug")
+
+    import application.parser.file.pdf_trust as pt
+
+    monkeypatch.setattr(pt, "verify_pdf_file", _explode)
+    _fake_anydoc(monkeypatch, lambda path: "# converted fine")
+    path = tmp_path / "x.pdf"
+    path.write_bytes(b"%PDF-1.4 irrelevant")
+
+    assert AnydocParser().parse_file(path) == "# converted fine"
+
+
+def test_tableize_applied_when_enabled(monkeypatch, tmp_path):
+    from application.parser.file import anydoc_parser as ap
+
+    monkeypatch.setattr(ap.settings, "ANYDOC_TABLEIZE", True)
+    monkeypatch.setattr(ap.settings, "PDF_TRUST_CHECK", False)
+    _fake_anydoc(
+        monkeypatch,
+        lambda path: "Cash ..... 1,234 900\nDebt ..... 2,000 1,500\nEquity ..... 900 800",
+    )
+    path = tmp_path / "x.pdf"
+    path.write_bytes(b"%PDF-1.4 irrelevant")
+
+    out = AnydocParser().parse_file(path)
+
+    assert "| Cash | 1,234 | 900 |" in out
+
+
+def test_tableize_disabled_by_default(monkeypatch, tmp_path):
+    from application.parser.file import anydoc_parser as ap
+
+    monkeypatch.setattr(ap.settings, "PDF_TRUST_CHECK", False)
+    flat = "Cash ..... 1,234 900\nDebt ..... 2,000 1,500\nEquity ..... 900 800"
+    _fake_anydoc(monkeypatch, lambda path: flat)
+    path = tmp_path / "x.pdf"
+    path.write_bytes(b"%PDF-1.4 irrelevant")
+
+    assert AnydocParser().parse_file(path) == flat
+
+
+def test_tableize_never_touches_docling_reroute(monkeypatch):
+    from application.parser.file import anydoc_parser as ap
+
+    monkeypatch.setattr(ap.settings, "ANYDOC_TABLEIZE", True)
+    fallback = _FakeDoclingFallback()
+    parser = AnydocParser(fallback_parser=fallback)
+
+    out = parser.parse_file(CID_PDF)
+
+    assert out == _REROUTE_TEXT  # verbatim, not post-processed
+
+
+# --- scanned-PDF near-empty guard (the OCR_ENGINE seam's loud-failure side) ------
+
+
+def test_scanned_pdf_with_near_empty_fallback_fails_loudly_when_ocr_off(tmp_path):
+    """A scan whose fallback (OCR off) extracts almost nothing must fail with
+    an actionable message, not be stored as an empty document."""
+    path = _scanned_pdf(tmp_path / "scan.pdf")
+    fallback = _RecordingFallback(result="   ")
+    fallback.ocr_enabled = False
+    parser = AnydocParser(fallback_parser=fallback)
+
+    with pytest.raises(DocumentParseError, match="OCR_ENABLED"):
+        parser.parse_file(path)
+    assert parser.last_engine is None
+
+
+def test_scanned_pdf_with_near_empty_fallback_and_ocr_on_reports_it(tmp_path):
+    path = _scanned_pdf(tmp_path / "scan.pdf")
+    fallback = _RecordingFallback(result="x")
+    fallback.ocr_enabled = True
+
+    with pytest.raises(DocumentParseError, match="even with OCR enabled"):
+        AnydocParser(fallback_parser=fallback).parse_file(path)
+
+
+def test_scanned_pdf_with_substantial_fallback_output_passes(tmp_path):
+    """docling extracting a text layer anydoc refused (the HK-bill case) must
+    keep working — the guard only fires on near-empty results."""
+    path = _scanned_pdf(tmp_path / "scan.pdf")
+    out = AnydocParser(fallback_parser=_RecordingFallback()).parse_file(path)
+    assert out == FALLBACK_TEXT
+
+
+def test_non_scan_refusals_do_not_trigger_the_guard(tmp_path, monkeypatch):
+    """Only the OCR-required refusal implies 'content exists but needs OCR';
+    a malformed file with a short fallback result stays a successful parse."""
+    fake = _fake_anydoc(monkeypatch, None)
+
+    class MalformedError(fake.ConvertError):
+        pass
+
+    fake.MalformedError = MalformedError
+
+    def _refuse(path):
+        raise MalformedError("structurally unusable")
+
+    fake.to_markdown = _refuse
+    path = tmp_path / "x.docx"
+    path.write_bytes(b"irrelevant")
+
+    out = AnydocParser(fallback_parser=_RecordingFallback(result="tiny")).parse_file(path)
+    assert out == "tiny"
+
+
+# --- spreadsheets: a resource limit delegates, other formats stay terminal ----
+
+
+def test_resource_limit_on_a_spreadsheet_delegates_to_the_tabular_fallback(tmp_path, monkeypatch):
+    """anydoc refuses a 265k-row sheet on fixed limits that openpyxl/pandas
+    read fine, so for tabular suffixes the limit is a reason to fall back."""
+    fake = _fake_anydoc(monkeypatch, None)
+
+    def _refuse(path):
+        raise fake.ResourceLimitError("resource limit exceeded (max_xml_nodes): 2000000")
+
+    fake.to_markdown = _refuse
+    fallback = _RecordingFallback()
+    parser = AnydocParser(fallback_parser=fallback)
+    path = tmp_path / "big.xlsx"
+    path.write_bytes(b"PK")
+
+    assert parser.parse_file(path) == FALLBACK_TEXT
+    assert fallback.calls == [path]
+    assert parser.last_engine == "_RecordingFallback"
+
+
+def test_resource_limit_on_a_spreadsheet_without_fallback_is_terminal(tmp_path, monkeypatch):
+    fake = _fake_anydoc(monkeypatch, None)
+
+    def _refuse(path):
+        raise fake.ResourceLimitError("resource limit exceeded (max_entry_bytes)")
+
+    fake.to_markdown = _refuse
+    path = tmp_path / "big.xlsx"
+    path.write_bytes(b"PK")
+
+    with pytest.raises(DocumentParseError, match="max_entry_bytes"):
+        AnydocParser().parse_file(path)
+
+
+# --- mixed documents: one page's OCR failure costs that page only ------------
+
+
+class _PerPageOcrFallback(BaseParser):
+    ocr_enabled = True
+
+    def __init__(self, failing_index):
+        super().__init__()
+        self.failing_index = failing_index
+        self.requests = []
+
+    def _init_parser(self):
+        return {}
+
+    def parse_file(self, file, errors="ignore"):
+        return "unused"
+
+    def ocr_pages(self, file, indices):
+        self.requests.append(list(indices))
+        if self.failing_index in indices:
+            raise DocumentParseError("tesseract failed on this page")
+        return {index: f"OCR TEXT PAGE {index + 1}" for index in indices}
+
+
+def test_one_failing_page_does_not_drop_the_other_pages_ocr(tmp_path, monkeypatch):
+    pytest.importorskip("pypdfium2")
+    pytest.importorskip("pypdf")
+    _fake_anydoc(monkeypatch, lambda path: "Text pages read by anydoc.")
+    fallback = _PerPageOcrFallback(failing_index=0)
+    parser = AnydocParser(fallback_parser=fallback)
+    path = _scanned_pdf(tmp_path / "mixed.pdf", pages=3)  # every page probes as scanned
+
+    text = parser.parse_file(path)
+
+    assert fallback.requests == [[0], [1], [2]]
+    assert "OCR TEXT PAGE 1" not in text
+    assert "OCR TEXT PAGE 2" in text and "OCR TEXT PAGE 3" in text
+    assert text.startswith("Text pages read by anydoc.")
+    # (The font-less blank-page fixture also earns a trust-check warning.)
+    assert parser.get_file_metadata(path)["ocr_pages"] == 2

tests/parser/file/test_bulk.py

@@ -508,3 +508,257 @@ class TestGetDefaultFileExtractor:
             mock_fn.return_value = {".pdf": MagicMock(), ".md": MagicMock()}
             result = mock_fn()
             assert ".pdf" in result
+
+
+# =====================================================================
+# DOC_PARSER_ENGINE switch
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestParserEngineSwitch:
+    """``get_default_file_extractor`` honours ``DOC_PARSER_ENGINE`` / ``engine``."""
+
+    @pytest.fixture
+    def settings(self):
+        from application.core.settings import settings
+
+        return settings
+
+    @pytest.fixture(autouse=True)
+    def _default_ocr_settings(self, monkeypatch):
+        """Pin the OCR knobs to their defaults so a developer's ``.env`` (OCR on,
+        native backend) cannot change which parsers the maps hand out here."""
+        from application.core.settings import settings
+
+        monkeypatch.setattr(settings, "OCR_ENABLED", False)
+        monkeypatch.setattr(settings, "OCR_BACKEND", "auto")
+
+    def test_default_engine_is_anydoc(self, settings, monkeypatch):
+        pytest.importorskip("anydoc")
+        from application.parser.file.anydoc_parser import ANYDOC_SUFFIXES, AnydocParser
+        from application.parser.file.bulk import get_default_file_extractor
+        from application.parser.file.html_parser import HTMLMarkdownParser
+
+        monkeypatch.setattr(settings, "DOC_PARSER_ENGINE", "anydoc")
+        extractor = get_default_file_extractor()
+
+        for suffix in ANYDOC_SUFFIXES:
+            assert isinstance(extractor[suffix], AnydocParser), suffix
+        assert isinstance(extractor[".html"], HTMLMarkdownParser)
+        assert isinstance(extractor[".xhtml"], HTMLMarkdownParser)
+        # Specialised parsers are untouched by the engine choice.
+        assert type(extractor[".md"]).__name__ == "MarkdownParser"
+        assert type(extractor[".json"]).__name__ == "JSONParser"
+        assert type(extractor[".epub"]).__name__ == "EpubParser"
+
+    def test_anydoc_falls_back_to_docling_when_installed(self):
+        pytest.importorskip("anydoc")
+        pytest.importorskip("docling")
+        from application.parser.file.bulk import get_default_file_extractor
+
+        extractor = get_default_file_extractor(engine="anydoc")
+
+        assert type(extractor[".pdf"].fallback_parser).__name__ == "DoclingPDFParser"
+        assert type(extractor[".csv"].fallback_parser).__name__ == "DoclingCSVParser"
+        assert extractor[".doc"].fallback_parser is None  # nothing else reads .doc
+        # docling keeps the formats anydoc cannot read.
+        assert type(extractor[".vtt"]).__name__ == "DoclingVTTParser"
+
+    def test_anydoc_fallback_honours_ocr_flag(self):
+        pytest.importorskip("anydoc")
+        pytest.importorskip("docling")
+        from application.parser.file.bulk import get_default_file_extractor
+
+        extractor = get_default_file_extractor(engine="anydoc", ocr_enabled=True)
+
+        assert extractor[".pdf"].fallback_parser.ocr_enabled is True
+        assert type(extractor[".png"]).__name__ == "DoclingImageParser"
+
+    def test_anydoc_falls_back_to_legacy_without_docling(self, monkeypatch):
+        pytest.importorskip("anydoc")
+        import sys
+
+        from application.parser.file.bulk import get_default_file_extractor
+
+        monkeypatch.setitem(sys.modules, "docling", None)
+        extractor = get_default_file_extractor(engine="anydoc")
+
+        assert type(extractor[".pdf"].fallback_parser).__name__ == "PDFParser"
+        assert type(extractor[".xlsx"].fallback_parser).__name__ == "ExcelParser"
+        assert type(extractor[".png"]).__name__ == "ImageParser"
+
+    def test_docling_engine_keeps_docling_map(self):
+        pytest.importorskip("docling")
+        from application.parser.file.bulk import get_default_file_extractor
+
+        extractor = get_default_file_extractor(engine="docling")
+
+        assert type(extractor[".pdf"]).__name__ == "DoclingPDFParser"
+        assert type(extractor[".html"]).__name__ == "DoclingHTMLParser"
+
+    def test_setting_selects_docling(self, settings, monkeypatch):
+        pytest.importorskip("docling")
+        from application.parser.file.bulk import get_default_file_extractor
+
+        monkeypatch.setattr(settings, "DOC_PARSER_ENGINE", "docling")
+        assert type(get_default_file_extractor()[".pdf"]).__name__ == "DoclingPDFParser"
+
+    def test_unknown_engine_uses_anydoc(self):
+        pytest.importorskip("anydoc")
+        from application.parser.file.anydoc_parser import AnydocParser
+        from application.parser.file.bulk import get_default_file_extractor
+
+        assert isinstance(get_default_file_extractor(engine="ghost")[".pdf"], AnydocParser)
+
+    def test_missing_anydoc_degrades_to_base_engine(self, monkeypatch):
+        import sys
+
+        from application.parser.file.bulk import get_default_file_extractor
+
+        monkeypatch.setitem(sys.modules, "anydoc", None)
+        extractor = get_default_file_extractor(engine="anydoc")
+
+        assert type(extractor[".pdf"]).__name__ in ("DoclingPDFParser", "PDFParser")
+        assert type(extractor[".html"]).__name__ in ("DoclingHTMLParser", "HTMLParser")
+
+    def test_missing_docling_map_is_usable(self, monkeypatch):
+        """Regression: with docling absent the map handed out docling parsers whose
+        ``init_parser`` raised ``ImportError`` — not a ``DocumentParseError`` — so
+        the first file aborted the whole ingest instead of degrading."""
+        import sys
+
+        from application.parser.file.bulk import get_default_file_extractor
+
+        monkeypatch.setitem(sys.modules, "docling", None)
+        extractor = get_default_file_extractor(engine="docling")
+
+        assert type(extractor[".pdf"]).__name__ == "PDFParser"
+        assert ".xhtml" in extractor
+        for suffix in (".pdf", ".docx", ".csv", ".xlsx", ".html", ".xhtml", ".pptx"):
+            extractor[suffix].init_parser()
+
+
+# =====================================================================
+# Gained formats (anydoc-only: legacy/macro Office, OpenDocument, RTF)
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestGainedFormats:
+    def test_every_supported_extension_has_a_parser_under_both_engines(self):
+        """The invariant that keeps constants.py and the parser maps in lockstep.
+
+        Without it, an extension accepted at upload but missing from the map
+        falls to SimpleDirectoryReader's plain-text read — for a binary
+        format that means mojibake silently ingested and embedded.
+        (`.txt` is the one deliberate plain-text read.)
+        """
+        pytest.importorskip("anydoc")
+        from application.parser.file.bulk import get_default_file_extractor
+        from application.parser.file.constants import (
+            SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS,
+        )
+
+        for engine in ("anydoc", "docling"):
+            extractor = get_default_file_extractor(engine=engine)
+            missing = [
+                suffix
+                for suffix in SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS
+                if suffix != ".txt" and suffix not in extractor
+            ]
+            assert missing == [], (engine, missing)
+
+    def test_gained_formats_map_to_anydoc_under_both_engines(self):
+        pytest.importorskip("anydoc")
+        from application.parser.file.anydoc_parser import (
+            ANYDOC_GAINED_SUFFIXES,
+            AnydocParser,
+        )
+        from application.parser.file.bulk import get_default_file_extractor
+
+        for engine in ("anydoc", "docling"):
+            extractor = get_default_file_extractor(engine=engine)
+            for suffix in ANYDOC_GAINED_SUFFIXES:
+                assert isinstance(extractor[suffix], AnydocParser), (engine, suffix)
+
+    def test_gained_formats_never_fall_back_to_anydoc_itself(self):
+        pytest.importorskip("anydoc")
+        from application.parser.file.anydoc_parser import AnydocParser
+        from application.parser.file.bulk import get_default_file_extractor
+
+        extractor = get_default_file_extractor(engine="anydoc")
+        assert extractor[".doc"].fallback_parser is None
+        assert extractor[".rtf"].fallback_parser is None
+        # ...while the core five keep their real fallback.
+        assert extractor[".pdf"].fallback_parser is not None
+        assert not isinstance(extractor[".pdf"].fallback_parser, AnydocParser)
+
+    def test_gained_entries_absent_without_anydoc(self, monkeypatch):
+        import sys
+
+        from application.parser.file.bulk import get_default_file_extractor
+
+        monkeypatch.setitem(sys.modules, "anydoc", None)
+        extractor = get_default_file_extractor(engine="docling")
+        assert ".doc" not in extractor  # degrades to the pre-anydoc map
+
+    def test_rtf_converts_end_to_end(self, tmp_path):
+        pytest.importorskip("anydoc")
+        from application.parser.file.bulk import get_default_file_extractor
+
+        path = tmp_path / "note.rtf"
+        path.write_text(r"{\rtf1\ansi Hello {\b bold} world.\par Second paragraph.}")
+        parser = get_default_file_extractor()[".rtf"]
+        parser.init_parser()
+
+        out = parser.parse_file(path)
+
+        assert "Hello **bold** world." in out
+        assert "Second paragraph." in out
+
+    def test_odt_converts_end_to_end(self, tmp_path):
+        pytest.importorskip("anydoc")
+        import zipfile
+
+        from application.parser.file.bulk import get_default_file_extractor
+
+        path = tmp_path / "doc.odt"
+        with zipfile.ZipFile(path, "w") as z:
+            z.writestr(
+                "mimetype",
+                "application/vnd.oasis.opendocument.text",
+                compress_type=zipfile.ZIP_STORED,
+            )
+            z.writestr(
+                "META-INF/manifest.xml",
+                '<?xml version="1.0"?><manifest:manifest xmlns:manifest="urn:oasis:names:tc:opendocument:xmlns:manifest:1.0">'
+                '<manifest:file-entry manifest:full-path="/" manifest:media-type="application/vnd.oasis.opendocument.text"/>'
+                '<manifest:file-entry manifest:full-path="content.xml" manifest:media-type="text/xml"/></manifest:manifest>',
+            )
+            z.writestr(
+                "content.xml",
+                '<?xml version="1.0"?><office:document-content xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" '
+                'xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0"><office:body><office:text>'
+                '<text:h text:outline-level="1">Title</text:h><text:p>Body text here.</text:p>'
+                "</office:text></office:body></office:document-content>",
+            )
+        parser = get_default_file_extractor()[".odt"]
+
+        out = parser.parse_file(path)
+
+        assert "# Title" in out
+        assert "Body text here." in out
+
+
+def test_gained_suffix_without_a_parser_is_rejected_not_read_as_text(tmp_path):
+    """Without anydoc an OLE .doc must not be indexed as decoded binary garbage."""
+    from application.parser.file.base_parser import DocumentParseError
+    from application.parser.file.bulk import SimpleDirectoryReader
+
+    path = tmp_path / "legacy.doc"
+    path.write_bytes(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 64)
+    reader = SimpleDirectoryReader(input_files=[str(path)], file_extractor={".pdf": object()})
+
+    with pytest.raises(DocumentParseError, match="firecrawl-anydoc"):
+        reader.load_data()

tests/parser/file/test_docling_parser.py

@@ -1,6 +1,6 @@
 """Comprehensive tests for application/parser/file/docling_parser.py
 
-Covers: DoclingParser (init, _init_parser, _get_ocr_options, _export_content,
+Covers: DoclingParser (init, _init_parser, OCR engine selection, _export_content,
 parse_file), subclass initialization, error handling.
 """
 
@@ -27,8 +27,8 @@ class TestDoclingParserInit:
         assert parser.ocr_enabled is True
         assert parser.table_structure is True
         assert parser.export_format == "markdown"
-        assert parser.use_rapidocr is True
-        assert parser.ocr_languages == ["english"]
+        assert parser.ocr_engine is None  # None -> settings.OCR_ENGINE at build
+        assert parser.ocr_languages is None  # None -> the engine's own default
         assert parser.force_full_page_ocr is False
         assert parser._converter is None
 
@@ -39,14 +39,14 @@ class TestDoclingParserInit:
             ocr_enabled=False,
             table_structure=False,
             export_format="text",
-            use_rapidocr=False,
+            ocr_engine="rapidocr",
             ocr_languages=["german"],
             force_full_page_ocr=True,
         )
         assert parser.ocr_enabled is False
         assert parser.table_structure is False
         assert parser.export_format == "text"
-        assert parser.use_rapidocr is False
+        assert parser.ocr_engine == "rapidocr"
         assert parser.ocr_languages == ["german"]
         assert parser.force_full_page_ocr is True
 
@@ -90,44 +90,182 @@ class TestDoclingParserInitParser:
 
 
 @pytest.mark.unit
-class TestGetOCROptions:
+class TestOcrEngineSelection:
+    """``OCR_ENGINE`` resolution and per-engine option building."""
 
-    def test_returns_none_when_rapidocr_disabled(self):
+    @pytest.fixture
+    def settings(self):
+        from application.core.settings import settings
+
+        return settings
+
+    def test_default_setting_is_tesseract(self):
+        # Field defaults, not the live settings: a developer's ``.env`` may
+        # legitimately set other engines/languages.
+        from application.core.settings import Settings
+
+        defaults = Settings.model_construct()
+        assert defaults.OCR_ENGINE == "tesseract"
+        assert defaults.OCR_LANGS == "eng"
+        assert defaults.OCR_BACKEND == "auto"
+
+    def test_none_reads_setting(self, settings, monkeypatch):
+        from application.parser.file.docling_parser import _resolve_ocr_engine
+
+        monkeypatch.setattr(settings, "OCR_ENGINE", "auto")
+        assert _resolve_ocr_engine(None) == "auto"
+
+    def test_unknown_engine_degrades_to_auto(self):
+        from application.parser.file.docling_parser import _resolve_ocr_engine
+
+        assert _resolve_ocr_engine("easyocr") == "auto"
+
+    def test_tesseract_without_binary_degrades_to_auto(self, monkeypatch):
+        from application.parser.file import docling_parser as dp
+
+        monkeypatch.setattr(dp.shutil, "which", lambda name: None)
+        assert dp._resolve_ocr_engine("tesseract") == "auto"
+
+    def test_tesseract_with_binary_selected(self, monkeypatch):
+        from application.parser.file import docling_parser as dp
+
+        monkeypatch.setattr(dp.shutil, "which", lambda name: "/usr/bin/tesseract")
+        assert dp._resolve_ocr_engine("tesseract") == "tesseract"
+
+    def test_ocrmac_off_darwin_degrades_to_auto(self, monkeypatch):
+        from application.parser.file import docling_parser as dp
+
+        monkeypatch.setattr(dp.sys, "platform", "linux")
+        assert dp._resolve_ocr_engine("ocrmac") == "auto"
+
+    def test_rapidocr_missing_degrades_to_auto(self, monkeypatch):
+        import sys
+
+        from application.parser.file import docling_parser as dp
+
+        monkeypatch.setitem(sys.modules, "rapidocr", None)
+        assert dp._resolve_ocr_engine("rapidocr") == "auto"
+
+    def test_deepseek_passes_through(self):
+        from application.parser.file.docling_parser import _resolve_ocr_engine
+
+        assert _resolve_ocr_engine("deepseek") == "deepseek"
+
+    def test_build_auto_returns_none(self):
+        from application.parser.file.docling_parser import _build_ocr_options
+
+        assert _build_ocr_options("auto", None, True) is None
+
+    def test_build_tesseract_reads_ocr_langs(self, settings, monkeypatch):
+        pytest.importorskip("docling")
+        import application.parser.file.ocr_parser as op
+        from application.parser.file.docling_parser import _build_ocr_options
+
+        # Pack inventory unknown: the resolved list is passed through untouched
+        # (a host with tesseract but no chi_sim pack would otherwise drop it).
+        monkeypatch.setattr(op, "tesseract_languages", lambda: None)
+        monkeypatch.setattr(settings, "OCR_LANGS", "eng+chi_sim")
+        options = _build_ocr_options("tesseract", None, True)
+
+        assert type(options).__name__ == "TesseractCliOcrOptions"
+        assert options.lang == ["eng", "chi_sim"]
+        assert options.force_full_page_ocr is True
+
+    def test_build_tesseract_explicit_languages_win(self, monkeypatch):
+        pytest.importorskip("docling")
+        import application.parser.file.ocr_parser as op
+        from application.parser.file.docling_parser import _build_ocr_options
+
+        # Pack inventory unknown: the requested list is passed through untouched.
+        monkeypatch.setattr(op, "tesseract_languages", lambda: None)
+        options = _build_ocr_options("tesseract", ["deu"], False)
+        assert options.lang == ["deu"]
+
+    def test_build_rapidocr(self):
+        pytest.importorskip("docling")
+        from application.parser.file.docling_parser import _build_ocr_options
+
+        options = _build_ocr_options("rapidocr", None, False)
+        assert type(options).__name__ == "RapidOcrOptions"
+        assert options.lang == ["english"]
+
+    def test_build_import_failure_returns_none(self, monkeypatch):
+        import sys
+
+        from application.parser.file.docling_parser import _build_ocr_options
+
+        monkeypatch.setitem(sys.modules, "docling.datamodel.pipeline_options", None)
+        assert _build_ocr_options("tesseract", ["eng"], False) is None
+        assert _build_ocr_options("ocrmac", None, False) is None
+
+    def test_build_generic_failure_returns_none(self, monkeypatch):
+        import sys
+        import types
+
+        from application.parser.file.docling_parser import _build_ocr_options
+
+        fake = types.ModuleType("docling.datamodel.pipeline_options")
+
+        def _boom(**kwargs):
+            raise RuntimeError("bad options")
+
+        fake.RapidOcrOptions = _boom
+        monkeypatch.setitem(sys.modules, "docling.datamodel.pipeline_options", fake)
+        assert _build_ocr_options("rapidocr", None, False) is None
+
+
+@pytest.mark.unit
+class TestDeepseekVlmConverter:
+    """OCR_ENGINE=deepseek swaps the whole pipeline for docling's VLM route."""
+
+    @pytest.fixture(autouse=True)
+    def _requires_docling(self):
+        pytest.importorskip("docling")
+
+    def test_deepseek_builds_vlm_converter(self, monkeypatch):
+        from application.core.settings import settings
         from application.parser.file.docling_parser import DoclingParser
 
-        parser = DoclingParser(use_rapidocr=False)
-        assert parser._get_ocr_options() is None
+        monkeypatch.setattr(settings, "OCR_DEEPSEEK_URL", "http://gpu-host:8000/v1/chat/completions")
+        monkeypatch.setattr(settings, "OCR_DEEPSEEK_MODEL", "deepseek-ocr-x")
 
-    def test_returns_options_when_available(self):
+        built = {}
+
+        def _capture_converter(format_options):
+            built["format_options"] = format_options
+            return MagicMock()
+
+        monkeypatch.setattr(
+            "docling.document_converter.DocumentConverter", _capture_converter
+        )
+        parser = DoclingParser(ocr_enabled=True, ocr_engine="deepseek")
+        parser._create_converter()
+
+        from docling.datamodel.base_models import InputFormat
+        from docling.pipeline.vlm_pipeline import VlmPipeline
+
+        pdf_option = built["format_options"][InputFormat.PDF]
+        image_option = built["format_options"][InputFormat.IMAGE]
+        assert pdf_option.pipeline_cls is VlmPipeline
+        assert image_option.pipeline_cls is VlmPipeline
+        vlm = pdf_option.pipeline_options.vlm_options
+        assert vlm.url == "http://gpu-host:8000/v1/chat/completions"
+        assert vlm.params["model"] == "deepseek-ocr-x"
+        assert pdf_option.pipeline_options.enable_remote_services is True
+
+    def test_deepseek_ignored_when_ocr_disabled(self, monkeypatch):
         from application.parser.file.docling_parser import DoclingParser
 
-        parser = DoclingParser(use_rapidocr=True, ocr_languages=["english"])
+        vlm_called = []
+        monkeypatch.setattr(
+            DoclingParser,
+            "_create_vlm_converter",
+            lambda self: vlm_called.append(True),
+        )
+        monkeypatch.setattr("docling.document_converter.DocumentConverter", MagicMock())
+        DoclingParser(ocr_enabled=False, ocr_engine="deepseek")._create_converter()
 
-        mock_options = MagicMock()
-        with patch(
-            "application.parser.file.docling_parser.DoclingParser._get_ocr_options",
-            return_value=mock_options,
-        ):
-            result = parser._get_ocr_options()
-            assert result is mock_options
-
-    def test_returns_none_on_import_error(self):
-        from application.parser.file.docling_parser import DoclingParser
-
-        parser = DoclingParser(use_rapidocr=True)
-
-        # Simulate the ImportError path
-        original = parser._get_ocr_options
-
-        def patched_get_ocr():
-            try:
-                raise ImportError("No RapidOcrOptions")
-            except ImportError:
-                return None
-
-        parser._get_ocr_options = patched_get_ocr
-        assert parser._get_ocr_options() is None
-        parser._get_ocr_options = original
+        assert vlm_called == []
 
 
 # =====================================================================
@@ -431,38 +569,57 @@ class TestDoclingSubclasses:
 
 @pytest.mark.unit
 class TestDoclingParserGaps:
-    def test_get_ocr_options_import_error_returns_none(self):
-        """Cover lines 148-150: ImportError returns None."""
-        from application.parser.file.docling_parser import DoclingParser
-
-        parser = DoclingParser(ocr_enabled=True, use_rapidocr=True)
-        with patch.dict("sys.modules", {"docling.datamodel.pipeline_options": None}):
-            # Force re-import to trigger ImportError
-            with patch(
-                "builtins.__import__", side_effect=ImportError("no module")
-            ):
-                result = parser._get_ocr_options()
-                assert result is None
-
-    def test_get_ocr_options_generic_error_returns_none(self):
-        """Cover lines 151-153: generic Exception returns None."""
-        from application.parser.file.docling_parser import DoclingParser
-
-        parser = DoclingParser(ocr_enabled=True, use_rapidocr=True)
-        with patch(
-            "builtins.__import__",
-            side_effect=RuntimeError("unexpected"),
-        ):
-            result = parser._get_ocr_options()
-            assert result is None
-
     def test_csv_parser_init(self):
         """Cover line 289: DoclingCSVParser.__init__ calls super."""
         from application.parser.file.docling_parser import DoclingCSVParser
 
         parser = DoclingCSVParser()
         assert parser.export_format == "markdown"
-        assert parser.ocr_enabled is True
+        assert parser.ocr_enabled is False
+
+
+@pytest.mark.unit
+class TestNonOcrParsersLeaveTextAlone:
+    """Office/markup docling parsers never OCR, so the tesseract CJK
+    post-processing must not touch their born-digital exports."""
+
+    @pytest.mark.parametrize(
+        "name",
+        [
+            "DoclingDocxParser",
+            "DoclingPPTXParser",
+            "DoclingXLSXParser",
+            "DoclingHTMLParser",
+            "DoclingCSVParser",
+            "DoclingMarkdownParser",
+            "DoclingAsciiDocParser",
+            "DoclingVTTParser",
+            "DoclingXMLParser",
+        ],
+    )
+    def test_ocr_is_off_by_construction(self, name):
+        from application.parser.file import docling_parser as dp
+
+        assert getattr(dp, name)().ocr_enabled is False
+
+    def test_cjk_spaces_survive_in_a_docx_export(self, monkeypatch):
+        from application.core.settings import settings
+        from application.parser.file.docling_parser import DoclingDocxParser
+
+        monkeypatch.setattr(settings, "OCR_ENGINE", "tesseract")
+        parser = DoclingDocxParser()
+        # Simulate what _create_converter records for an OCR-on parser; the
+        # docx parser is OCR-off so the collapse must stay inert regardless.
+        parser._active_ocr_engine = "tesseract"
+        text = "\u5f20 \u4e09 \u548c \u674e \u56db\n\uff21\uff22\uff23 \uff24\uff25\uff26"
+        assert parser._postprocess_ocr_text(text) == text
+
+    def test_pdf_parser_with_tesseract_still_collapses(self):
+        from application.parser.file.docling_parser import DoclingPDFParser
+
+        parser = DoclingPDFParser(ocr_enabled=True)
+        parser._active_ocr_engine = "tesseract"
+        assert parser._postprocess_ocr_text("\u4e92\u76f8 \u4fdd\u5bc6") == "\u4e92\u76f8\u4fdd\u5bc6"
 
 
 # =====================================================================
@@ -910,6 +1067,12 @@ class TestApplyInferenceSettings:
     and buys nothing for one-shot parses.
     """
 
+    @pytest.fixture(autouse=True)
+    def _requires_docling(self):
+        # These tests reach into docling.datamodel.settings for real; a base
+        # install (docling is an optional extra) skips them.
+        pytest.importorskip("docling")
+
     def test_disables_torch_compile_by_default(self, monkeypatch):
         from application.parser.file.docling_parser import _apply_inference_settings
 
@@ -1041,7 +1204,7 @@ def _set_threshold(monkeypatch, value: int) -> None:
             return getattr(real_settings, name)
 
     stub = _Stub()
-    stub.DOCLING_OCR_MIN_CHARS_PER_PAGE = value
+    stub.OCR_MIN_CHARS_PER_PAGE = value
     monkeypatch.setattr("application.core.settings.settings", stub)
 
 
@@ -1219,11 +1382,11 @@ class TestOCRDropoutGuard:
         assert create.call_count == 1
 
     def test_guard_does_not_apply_to_non_ocr_formats(self):
-        """DOCX parsers inherit ocr_enabled=True but never OCR anything."""
+        """DOCX parsers construct with OCR off: they never OCR anything."""
         from application.parser.file.docling_parser import DoclingDocxParser
 
         parser = DoclingDocxParser()
-        assert parser.ocr_enabled is True
+        assert parser.ocr_enabled is False
         converter = MagicMock()
         converter.convert.return_value = _mock_conversion("Hi", pages=1)
         parser._converter = converter
@@ -1412,6 +1575,12 @@ class TestOCRDropoutHelpers:
 class TestForceFullPageOCRWiring:
     """``force_full_page_ocr`` must reach the pipeline for any OCR engine."""
 
+    @pytest.fixture(autouse=True)
+    def _requires_docling(self):
+        # Builds real ``docling.datamodel.pipeline_options``; a base install
+        # (docling is an optional extra) skips these.
+        pytest.importorskip("docling")
+
     def _pipeline_options(self, monkeypatch, **kwargs):
         import docling.datamodel.pipeline_options as dpo
 
@@ -1430,11 +1599,11 @@ class TestForceFullPageOCRWiring:
         DoclingParser(**kwargs)._create_converter()
         return built[0]
 
-    def test_forced_without_rapidocr(self, monkeypatch):
+    def test_forced_with_default_auto_options(self, monkeypatch):
         options = self._pipeline_options(
             monkeypatch,
             ocr_enabled=True,
-            use_rapidocr=False,
+            ocr_engine="auto",
             force_full_page_ocr=True,
         )
         assert options.ocr_options.force_full_page_ocr is True
@@ -1443,7 +1612,101 @@ class TestForceFullPageOCRWiring:
         options = self._pipeline_options(
             monkeypatch,
             ocr_enabled=True,
-            use_rapidocr=False,
+            ocr_engine="auto",
             force_full_page_ocr=False,
         )
         assert options.ocr_options.force_full_page_ocr is False
+
+
+@pytest.mark.unit
+class TestDoclingTesseractPostprocessing:
+    """docling's tesseract path gets the same CJK glyph-space cleanup as the native engine."""
+
+    def _parser_with_export(self, engine, ocr_enabled=True):
+        from application.parser.file.docling_parser import DoclingPDFParser
+
+        parser = DoclingPDFParser(ocr_enabled=ocr_enabled)
+        parser._active_ocr_engine = engine
+        document = MagicMock()
+        document.export_to_markdown.return_value = "## 互相 保密 协议\n\nhello world 你 好"
+        return parser, document
+
+    def test_tesseract_output_collapses_cjk_spaces(self):
+        parser, document = self._parser_with_export("tesseract")
+        assert parser._export_content(document) == "## 互相保密协议\n\nhello world 你好"
+
+    def test_other_engines_untouched(self):
+        parser, document = self._parser_with_export("deepseek")
+        assert parser._export_content(document) == "## 互相 保密 协议\n\nhello world 你 好"
+
+    def test_ocr_off_untouched(self):
+        parser, document = self._parser_with_export("tesseract", ocr_enabled=False)
+        assert parser._export_content(document) == "## 互相 保密 协议\n\nhello world 你 好"
+
+
+@pytest.mark.unit
+class TestDoclingOcrPages:
+    def test_converts_each_requested_page_with_page_range(self, tmp_path):
+        from application.parser.file.docling_parser import DoclingPDFParser
+
+        parser = DoclingPDFParser(ocr_enabled=True)
+        converter = MagicMock()
+
+        def convert(source, page_range=None):
+            result = MagicMock()
+            result.document.export_to_markdown.return_value = f"page {page_range[0]} text " * 5
+            return result
+
+        converter.convert.side_effect = convert
+        parser._converter = converter
+        pdf = tmp_path / "doc.pdf"
+        pdf.write_bytes(b"%PDF-1.4")
+
+        texts = parser.ocr_pages(pdf, [2, 0])
+
+        assert set(texts) == {2, 0}
+        assert texts[2].startswith("page 3 text")
+        assert texts[0].startswith("page 1 text")
+        assert [c.kwargs["page_range"] for c in converter.convert.call_args_list] == [(3, 3), (1, 1)]
+
+    def test_failure_is_a_parse_error(self, tmp_path):
+        from application.parser.file.base_parser import DocumentParseError
+        from application.parser.file.docling_parser import DoclingPDFParser
+
+        parser = DoclingPDFParser(ocr_enabled=True)
+        parser._converter = MagicMock()
+        parser._converter.convert.side_effect = RuntimeError("boom")
+        pdf = tmp_path / "doc.pdf"
+        pdf.write_bytes(b"%PDF-1.4")
+        with pytest.raises(DocumentParseError, match="page 1 of doc.pdf"):
+            parser.ocr_pages(pdf, [0])
+
+
+@pytest.mark.unit
+class TestTesseractLanguageFilter:
+    def test_uninstalled_packs_are_dropped_with_a_warning(self, monkeypatch, caplog):
+        pytest.importorskip("docling")
+        import application.parser.file.ocr_parser as op
+        from application.parser.file.docling_parser import _build_ocr_options
+
+        monkeypatch.setattr(op, "tesseract_languages", lambda: frozenset({"eng", "osd"}))
+        with caplog.at_level("WARNING"):
+            options = _build_ocr_options("tesseract", ["eng", "chi_sim"], False)
+        assert options.lang == ["eng"]
+        assert "chi_sim" in caplog.text
+
+    def test_all_packs_missing_falls_back_to_eng(self, monkeypatch):
+        pytest.importorskip("docling")
+        import application.parser.file.ocr_parser as op
+        from application.parser.file.docling_parser import _build_ocr_options
+
+        monkeypatch.setattr(op, "tesseract_languages", lambda: frozenset({"eng"}))
+        assert _build_ocr_options("tesseract", ["xyz"], False).lang == ["eng"]
+
+    def test_unknown_inventory_keeps_the_list(self, monkeypatch):
+        pytest.importorskip("docling")
+        import application.parser.file.ocr_parser as op
+        from application.parser.file.docling_parser import _build_ocr_options
+
+        monkeypatch.setattr(op, "tesseract_languages", lambda: None)
+        assert _build_ocr_options("tesseract", ["eng", "deu"], False).lang == ["eng", "deu"]

tests/parser/file/test_html_parser.py

@@ -54,3 +54,225 @@ def test_html_parser_metadata_without_title(tmp_path):
 
 def test_html_parser_metadata_unreadable_file(tmp_path):
     assert HTMLParser().get_file_metadata(tmp_path / "missing.html") == {}
+
+
+# --- HTMLMarkdownParser: the anydoc engine's HTML path ---------------------------
+
+from application.parser.file.html_parser import HTMLMarkdownParser, html_to_markdown  # noqa: E402
+
+RICH_HTML = """<html><head><title>Doc Title</title>
+<style>.x { color: red }</style><script>var secret = 1;</script></head>
+<body>
+<h1>Heading</h1>
+<p>See <a href="../ref.html#anchor">the ref</a> for details.</p>
+<table><tr><th>h1</th><th>h2</th></tr><tr><td>1</td><td>2</td></tr></table>
+<pre><code>typing.__name__, typing.__spec__.name</code></pre>
+</body></html>"""
+
+
+@pytest.fixture
+def rich_html_file(tmp_path):
+    path = tmp_path / "rich.html"
+    path.write_text(RICH_HTML)
+    return path
+
+
+def test_markdown_parser_keeps_structure(rich_html_file):
+    out = HTMLMarkdownParser().parse_file(rich_html_file)
+
+    assert isinstance(out, str)
+    assert "# Heading" in out
+    assert "[the ref](../ref.html#anchor)" in out  # links kept, hrefs verbatim
+    assert "| h1 | h2 |" in out and "| 1 | 2 |" in out  # GFM table
+    assert "typing.__name__, typing.__spec__.name" in out  # code byte-exact
+
+
+def test_markdown_parser_drops_non_content(rich_html_file):
+    out = HTMLMarkdownParser().parse_file(rich_html_file)
+
+    assert "var secret" not in out
+    assert "color: red" not in out
+    assert "Doc Title" not in out  # title is metadata, not body text
+
+
+def test_markdown_parser_reports_title_metadata(rich_html_file):
+    assert HTMLMarkdownParser().get_file_metadata(rich_html_file) == {"title": "Doc Title"}
+
+
+def test_markdown_parser_differs_from_plain_text_contract(rich_html_file):
+    """``HTMLParser`` stays plain text (the ``fast`` engine's contract); only the new class emits Markdown."""
+    plain = HTMLParser().parse_file(rich_html_file)
+    assert "# Heading" not in plain
+    assert "[the ref]" not in plain
+
+
+def test_markdown_parser_handles_xhtml(tmp_path):
+    path = tmp_path / "page.xhtml"
+    path.write_text(
+        '<?xml version="1.0" encoding="UTF-8"?>\n'
+        '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" '
+        '"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">\n'
+        '<html xmlns="http://www.w3.org/1999/xhtml"><head><title>X</title></head>'
+        "<body><h2>Section</h2><p>Body &amp; entities.</p></body></html>"
+    )
+    out = HTMLMarkdownParser().parse_file(path)
+    assert "## Section" in out
+    assert "Body & entities." in out
+
+
+def test_markdown_parser_init():
+    parser = HTMLMarkdownParser()
+    assert not parser.parser_config_set
+    parser.init_parser()
+    assert parser.parser_config_set
+
+
+def test_html_to_markdown_collapses_blank_runs():
+    out = html_to_markdown("<p>a</p><br/><br/><br/><br/><p>b</p>")
+    assert "\n\n\n" not in out
+    assert out.startswith("a") and out.endswith("b")
+
+
+# --- HTMLMarkdownParser: input bounding and decoding ------------------------------
+
+
+def test_markdown_parser_head_truncates_oversized_markup(tmp_path, monkeypatch):
+    """Markup past MARKUP_MAX_BYTES is not parsed: the soup+markdownify tree costs
+    ~50x the input, and the upload cap is 100 MB."""
+    from application.core.settings import settings
+
+    monkeypatch.setattr(settings, "MARKUP_MAX_BYTES", 600)
+    path = tmp_path / "big.html"
+    body = "".join(f"<p>para {i}</p>\n" for i in range(300))
+    path.write_text("<html><head><title>Big</title></head><body>\n" + body + "</body></html>")
+    assert path.stat().st_size > 600
+
+    out = HTMLMarkdownParser().parse_file(path)
+
+    assert "para 0" in out
+    assert "para 299" not in out
+    assert HTMLMarkdownParser().get_file_metadata(path) == {"title": "Big"}
+
+
+def test_markdown_parser_gate_disabled_reads_everything(tmp_path, monkeypatch):
+    from application.core.settings import settings
+
+    monkeypatch.setattr(settings, "MARKUP_MAX_BYTES", 0)
+    path = tmp_path / "big.html"
+    body = "".join(f"<p>para {i}</p>\n" for i in range(300))
+    path.write_text("<html><body>\n" + body + "</body></html>")
+
+    out = HTMLMarkdownParser().parse_file(path)
+
+    assert "para 299" in out
+
+
+def test_markdown_parser_honours_declared_charset(tmp_path):
+    """The file is handed to BeautifulSoup as bytes so ``<meta charset>`` wins
+    over the process locale (docling did the same; the text parser does not)."""
+    path = tmp_path / "cp1250.html"
+    path.write_bytes(
+        '<html><head><meta charset="windows-1250"><title>Plzeň</title></head>'
+        "<body><p>Jiří z Plzně</p></body></html>".encode("cp1250")
+    )
+
+    assert "Jiří z Plzně" in HTMLMarkdownParser().parse_file(path)
+    assert HTMLMarkdownParser().get_file_metadata(path) == {"title": "Plzeň"}
+
+
+def test_markdown_parser_metadata_unreadable_file(tmp_path):
+    assert HTMLMarkdownParser().get_file_metadata(tmp_path / "missing.html") == {}
+
+
+def test_markdown_parser_cut_inside_utf8_char_does_not_mojibake_the_page(tmp_path, monkeypatch):
+    """A byte cut mid-character makes strict UTF-8 fail; BeautifulSoup would then
+    retry as windows-1252, which *succeeds* on umlauts and garbles everything."""
+    from application.core.settings import settings
+
+    path = tmp_path / "minified.html"
+    body = "<p>Größe Übermaß schön für</p>" * 400  # no newlines: the cut is arbitrary
+    path.write_bytes(
+        ('<html><head><meta charset="utf-8"><title>Größe</title></head><body>' + body + "</body></html>")
+        .encode("utf-8")
+    )
+    for cap in (2001, 2002, 2003):  # one of these lands inside a 2-byte sequence
+        monkeypatch.setattr(settings, "MARKUP_MAX_BYTES", cap)
+        out = HTMLMarkdownParser().parse_file(path)
+        assert "Größe Übermaß schön für" in out, cap
+        assert "Ã" not in out, cap
+
+
+def test_trim_torn_utf8_tail():
+    from application.parser.file.html_parser import _trim_torn_utf8_tail as trim
+
+    e_acute, snowman, emoji = "é".encode(), "☃".encode(), "😀".encode()
+    assert trim(b"abc") == b"abc"
+    assert trim(b"abc" + e_acute) == b"abc" + e_acute  # complete 2-byte
+    assert trim(b"abc" + e_acute[:1]) == b"abc"  # torn 2-byte
+    assert trim(b"abc" + snowman[:2]) == b"abc"  # torn 3-byte
+    assert trim(b"abc" + emoji[:3]) == b"abc"  # torn 4-byte
+    assert trim(b"abc" + emoji) == b"abc" + emoji
+    assert trim(b"") == b""
+    assert trim(b"\xe8") == b""  # lone lead byte (also a cp1250 letter: at most one glyph lost)
+
+
+def test_markdown_parser_metadata_reuses_last_parse(tmp_path, rich_html_file, monkeypatch):
+    """The metadata call after parse_file must not build the soup again."""
+    from application.parser.file import html_parser as mod
+
+    parser = HTMLMarkdownParser()
+    parser.parse_file(rich_html_file)
+    calls = []
+    monkeypatch.setattr(mod, "read_markup_head", lambda *a, **k: calls.append(a) or b"")
+    assert parser.get_file_metadata(rich_html_file) == {"title": "Doc Title"}
+    assert calls == []
+    # A different file is read afresh.
+    other = tmp_path / "other.html"
+    other.write_text("<title>Other</title>")
+    parser.get_file_metadata(other)
+    assert len(calls) == 1
+
+
+# --- second-pass fixes: XML prolog, data URIs, UTF-16 heads ---------------------
+
+
+def test_xml_prolog_does_not_leak_into_markdown(tmp_path):
+    from application.parser.file.html_parser import HTMLMarkdownParser
+
+    path = tmp_path / "doc.xhtml"
+    path.write_bytes(
+        b'<?xml version="1.0" encoding="UTF-8"?>\n<!DOCTYPE html>\n'
+        b'<html xmlns="http://www.w3.org/1999/xhtml"><head><title>T</title></head>'
+        b"<body><h1>Hi</h1><p>para</p><![CDATA[raw]]></body></html>"
+    )
+    text = HTMLMarkdownParser().parse_file(path)
+    assert "xml version" not in text
+    assert "raw" not in text
+    assert text.startswith("# Hi")
+
+
+def test_data_uris_are_stripped_from_images_and_links():
+    payload = "data:image/png;base64," + "A" * 200_000
+    html = (
+        f'<p>before</p><img src="{payload}" alt="chart"><a href="{payload}">dl</a>'
+        f'<img srcset="{payload} 1x" alt="x"><p>after</p>'
+    )
+    text = html_to_markdown(html)
+    assert "AAAA" not in text
+    assert "before" in text and "after" in text
+    assert len(text) < 200
+
+
+def test_utf16_head_keeps_an_even_byte_count(tmp_path, monkeypatch):
+    from application.core.settings import settings
+    from application.parser.file.html_parser import HTMLMarkdownParser, read_markup_head
+
+    body = "".join(f"<p>Zeile {i} Über Größe</p>\n" for i in range(200))
+    path = tmp_path / "wide.html"
+    path.write_bytes(("<html><body>" + body + "</body></html>").encode("utf-16"))  # BOM-prefixed
+    head = read_markup_head(path, 3001)
+    assert len(head) % 2 == 0
+    monkeypatch.setattr(settings, "MARKUP_MAX_BYTES", 3001)
+    text = HTMLMarkdownParser().parse_file(path)
+    assert "Zeile 0 Über Größe" in text
+    assert "\x00" not in text

tests/parser/file/test_pdf_trust.py

@@ -0,0 +1,254 @@
+"""Tests for the PDF trust checks behind ``PDF_TRUST_CHECK``.
+
+The checks target the two classes where anydoc drops text *silently*:
+composite (Type0) fonts without a ToUnicode map, and CJK-declaring PDFs
+whose extracted text carries almost no CJK. The scanner is byte-level and
+regex-based, so the synthetic fixtures below are minimal PDF fragments, not
+well-formed files; the one real fixture is the corpus document that loses
+its whole Chinese column in anydoc with no error.
+"""
+import zlib
+from pathlib import Path
+
+from application.parser.file.pdf_trust import (
+    check_pdf_fonts,
+    verify_extraction,
+    verify_pdf_file,
+)
+
+FIXTURES = Path(__file__).parent / "fixtures"
+
+
+def _obj(body: bytes) -> bytes:
+    return b"1 0 obj" + body + b"endobj\n"
+
+
+def _stream(payload: bytes) -> bytes:
+    return b"2 0 obj<</Filter/FlateDecode>>stream\n" + zlib.compress(payload) + b"\nendstream endobj\n"
+
+
+TYPE0_WITH_TOUNICODE = _obj(
+    b"<</Type/Font/Subtype/Type0/BaseFont/AAAAAA+Simple/ToUnicode 9 0 R>>"
+)
+TYPE0_BARE = _obj(b"<</Type/Font/Subtype/Type0/BaseFont/AAAAAA+Bare>>")
+SIMPLE_FONT = _obj(b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>")
+GB1_ORDERING = _obj(b"<</Registry(Adobe)/Ordering(GB1)/Supplement 5>>")
+
+
+class TestCheckPdfFonts:
+    def test_type0_with_tounicode_not_flagged(self):
+        result = check_pdf_fonts(TYPE0_WITH_TOUNICODE)
+        assert result["type0"] == 1
+        assert result["type0_no_tounicode"] == 0
+        assert not result["flagged"]
+
+    def test_type0_without_tounicode_flagged(self):
+        result = check_pdf_fonts(TYPE0_BARE)
+        assert result["type0_no_tounicode"] == 1
+        assert result["flagged"]
+
+    def test_simple_font_not_flagged(self):
+        result = check_pdf_fonts(SIMPLE_FONT)
+        assert result["type0"] == 0
+        assert result["has_fonts"]
+        assert not result["flagged"]
+
+    def test_no_fonts_flagged(self):
+        result = check_pdf_fonts(_obj(b"<</Type/Page>>"))
+        assert not result["has_fonts"]
+        assert result["flagged"]
+
+    def test_type0_inside_object_stream_is_seen(self):
+        """Object streams hold dicts with no obj markers; the scan must decompress them."""
+        data = _stream(b"<</Type/Font/Subtype/Type0/BaseFont/BBBBBB+Packed>>")
+        result = check_pdf_fonts(data)
+        assert result["type0"] == 1
+        assert result["type0_no_tounicode"] == 1
+        assert result["flagged"]
+
+    def test_font_inside_object_stream_counts_as_has_fonts(self):
+        """A fully-compressed PDF with only simple fonts must not false-positive."""
+        data = _stream(b"<</Type/Font/Subtype/TrueType/BaseFont/Arial>>")
+        result = check_pdf_fonts(data)
+        assert result["has_fonts"]
+        assert not result["flagged"]
+
+    def test_cjk_ordering_detected_in_stream(self):
+        data = _stream(b"<</Registry(Adobe)/Ordering(CNS1)/Supplement 4>>") + SIMPLE_FONT
+        assert check_pdf_fonts(data)["expects_cjk"]
+
+    def test_cjk_font_names_alone_do_not_set_expectation(self):
+        """Regression for the Berkshire false positive: Word-exported Latin PDFs
+        embed MS-Gothic subsets for stray full-width characters, and substring
+        matching also hits FranklinGothic — neither means CJK *content*, and a
+        false expectation reroutes a 150-page English report to the heavy
+        engine. Only CIDSystemInfo /Ordering may set the expectation."""
+        data = _obj(
+            b"<</Type/Font/Subtype/TrueType/BaseFont/AKBHWD+MS-Gothic>>"
+        ) + _obj(b"<</Type/Font/Subtype/TrueType/BaseFont/KMBGKH+FranklinGothic-Roman>>")
+        result = check_pdf_fonts(data)
+        assert not result["expects_cjk"]
+        assert not result["flagged"]
+
+
+class TestVerifyExtraction:
+    def test_clean_pdf_and_output(self):
+        assert verify_extraction(SIMPLE_FONT, "Plain English text.") == []
+
+    def test_bare_type0_reported(self):
+        problems = verify_extraction(TYPE0_BARE + SIMPLE_FONT, "some text")
+        assert len(problems) == 1
+        assert "ToUnicode" in problems[0]
+
+    def test_cjk_expected_but_missing(self):
+        problems = verify_extraction(GB1_ORDERING + SIMPLE_FONT, "English only output")
+        assert len(problems) == 1
+        assert "CJK" in problems[0]
+
+    def test_cjk_expected_and_present(self):
+        text = "标题：本协议由双方共同签署生效" # 14 CJK chars
+        assert verify_extraction(GB1_ORDERING + SIMPLE_FONT, text) == []
+
+    def test_both_problems_reported(self):
+        problems = verify_extraction(TYPE0_BARE + GB1_ORDERING, "english")
+        assert len(problems) == 2
+
+
+class TestRealFixture:
+    """The corpus PDF where anydoc silently drops the entire Chinese column."""
+
+    def test_cid_font_nda_is_flagged(self):
+        data = (FIXTURES / "nda_en_zh_cid_font.pdf").read_bytes()
+        result = check_pdf_fonts(data)
+        assert result["type0_no_tounicode"] >= 1
+        assert result["expects_cjk"]
+        assert result["flagged"]
+
+    def test_cjkless_extraction_reports_both(self):
+        data = (FIXTURES / "nda_en_zh_cid_font.pdf").read_bytes()
+        problems = verify_extraction(data, "MUTUAL NON-DISCLOSURE AGREEMENT ...")
+        assert any("ToUnicode" in p for p in problems)
+        assert any("CJK" in p for p in problems)
+
+
+def test_flate_bomb_streams_are_capped():
+    """Flate reaches ~1000:1, so per-stream inflation must be capped — an
+    uncapped decompress of a crafted PDF would OOM the ingest worker."""
+    from application.parser.file.pdf_trust import _decompressed_streams, _STREAM_INFLATE_CAP
+
+    bomb = _stream(b"\0" * (_STREAM_INFLATE_CAP * 4))
+    chunks = list(_decompressed_streams(bomb))
+    assert chunks
+    assert all(len(chunk) <= _STREAM_INFLATE_CAP for chunk in chunks)
+    # The capped scan still completes and reports sanely alongside real objects.
+    assert not check_pdf_fonts(SIMPLE_FONT + bomb)["flagged"]
+
+
+def test_verify_pdf_file_unreadable_trusts_output(tmp_path):
+    assert verify_pdf_file(tmp_path / "missing.pdf", "text") == []
+
+
+# --- stream scanning is a keyword walk, not a regex -----------------------------
+
+
+class TestStreamWalk:
+    def test_stream_without_eol_before_endstream_is_seen(self):
+        body = zlib.compress(b"<</Type/Font/Subtype/Type0/BaseFont/X>>")
+        data = b"2 0 obj<</Filter/FlateDecode>>stream\n" + body + b"endstream endobj\n"
+        assert check_pdf_fonts(data)["type0"] == 1
+
+    def test_undecodable_stream_does_not_swallow_the_next_one(self):
+        """A non-Flate (image) stream before a font stream must not merge the two."""
+        image = b"3 0 obj<</Filter/DCTDecode>>stream\n\xff\xd8 not zlib at all\nendstream endobj\n"
+        font = _stream(b"<</Type/Font/Subtype/Type0/BaseFont/X/ToUnicode 9 0 R>>")
+        result = check_pdf_fonts(image + font)
+        assert result["type0"] == 1
+        assert result["type0_no_tounicode"] == 0
+
+    def test_crlf_streams_decompress(self):
+        body = zlib.compress(b"/Ordering (GB1)")
+        data = b"2 0 obj<</Filter/FlateDecode>>stream\r\n" + body + b"\r\nendstream endobj\n"
+        assert check_pdf_fonts(data)["expects_cjk"] is True
+
+    def test_large_file_with_many_streams_is_linear(self):
+        """1000 undecodable streams: the old regex went quadratic here."""
+        import time
+
+        chunk = b"9 0 obj<</Length 30>>stream\n" + b"\x00" * 30 + b"endstream endobj\n"
+        data = chunk * 1000 + TYPE0_BARE
+        started = time.perf_counter()
+        result = check_pdf_fonts(data)
+        assert time.perf_counter() - started < 1.0
+        assert result["type0_no_tounicode"] == 1
+
+
+class TestObjectWalk:
+    def test_bodies_match_the_regex_reference(self):
+        """The forward walk yields exactly what ``\\d+\\s+\\d+\\s+obj(.*?)endobj`` did."""
+        import re
+
+        from application.parser.file.pdf_trust import _object_bodies
+
+        reference = re.compile(rb"\d+\s+\d+\s+obj(.*?)endobj", re.DOTALL)
+        data = (
+            TYPE0_BARE
+            + b"7 0 obj no terminator here "  # orphan header: body runs to the next endobj
+            + SIMPLE_FONT
+            + b"junk 12 0 obj<</A/B>>endobj"
+            + _stream(b"x")
+            + b"99 0 obj trailing orphan"
+        )
+        assert list(_object_bodies(data)) == [m.group(1) for m in reference.finditer(data)]
+
+    def test_orphan_headers_are_linear(self):
+        """Headers with no ``endobj`` after them: the old regex rescanned to EOF for
+        each one (2000 in 1 MB took 13.5 s), pinning an ingest worker per upload."""
+        import time
+
+        data = TYPE0_BARE + b"1 0 obj\n" * 20_000 + b"\x00" * 500_000
+        started = time.perf_counter()
+        result = check_pdf_fonts(data)
+        assert time.perf_counter() - started < 1.0
+        assert result["type0_no_tounicode"] == 1
+
+
+# --- the CJK cross-check is confirmed against pdfium's text layer --------------
+
+
+def _latin_pdf(path: Path) -> Path:
+    import pytest
+
+    pytest.importorskip("reportlab")
+    from reportlab.pdfgen import canvas
+
+    c = canvas.Canvas(str(path))
+    c.drawString(72, 700, "Plain English content, nothing else on the page.")
+    c.showPage()
+    c.save()
+    return path
+
+
+def test_stray_cjk_ordering_without_cjk_text_is_not_flagged(tmp_path):
+    """A Quartz-style export declares Japan1 on one unused font: pdfium sees no
+    CJK either, so anydoc dropped nothing and the file must not be rerouted."""
+    import pytest
+
+    pytest.importorskip("pypdfium2")
+    path = _latin_pdf(tmp_path / "latin.pdf")
+    with open(path, "ab") as fh:
+        fh.write(b"\n% trailing junk\n" + _obj(b"<</Registry(Adobe)/Ordering(Japan1)/Supplement 6>>"))
+    markdown = "Plain English content, nothing else on the page."
+
+    # The byte-level check alone still fires...
+    assert any(p.startswith("PDF declares CJK fonts") for p in verify_extraction(path.read_bytes(), markdown))
+    # ...and the file-level check withdraws it.
+    assert [p for p in verify_pdf_file(path, markdown) if p.startswith("PDF declares CJK fonts")] == []
+
+
+def test_real_cjk_drop_survives_the_cross_check():
+    import pytest
+
+    pytest.importorskip("pypdfium2")
+    fixture = FIXTURES / "nda_en_zh_cid_font.pdf"
+    problems = verify_pdf_file(fixture, "English only, the Chinese column is gone.")
+    assert any(p.startswith("PDF declares CJK fonts") for p in problems)

tests/parser/file/test_pdfium_parser.py

@@ -153,9 +153,24 @@ def test_threshold_sends_sparse_text_to_fallback(tmp_path, parser_cls):
 
 
 # --- extractor wiring: the guard that source ingestion keeps docling ---------
+# (under the docling engine; the anydoc engine ignores the fast path, see below)
 
 
-def test_extractor_defaults_to_docling_for_pdf():
+@pytest.fixture
+def docling_engine(monkeypatch):
+    # The wiring assertions below expect the docling map; on a base install
+    # (docling is an optional extra) the engine degrades to the legacy
+    # parsers and these tests do not apply.
+    pytest.importorskip("docling")
+    from application.core.settings import settings
+
+    monkeypatch.setattr(settings, "DOC_PARSER_ENGINE", "docling")
+    # A developer .env with OCR on (and OCR_BACKEND=native) would otherwise
+    # swap the native OCR parser in for the docling one these tests expect.
+    monkeypatch.setattr(settings, "OCR_ENABLED", False)
+
+
+def test_extractor_defaults_to_docling_for_pdf(docling_engine):
     """Sources must be unaffected: no fast path unless explicitly requested."""
     from application.parser.file.bulk import get_default_file_extractor
 
@@ -164,7 +179,7 @@ def test_extractor_defaults_to_docling_for_pdf():
     assert type(pdf_parser).__name__ == "DoclingPDFParser"
 
 
-def test_extractor_uses_fast_path_when_requested():
+def test_extractor_uses_fast_path_when_requested(docling_engine):
     from application.parser.file.bulk import get_default_file_extractor
     from application.parser.file.pdfium_parser import PdfiumTextParser
 
@@ -174,7 +189,20 @@ def test_extractor_uses_fast_path_when_requested():
     assert type(pdf_parser.fallback_parser).__name__ == "DoclingPDFParser"
 
 
-def test_fast_path_does_not_change_non_pdf_parsers():
+def test_anydoc_engine_ignores_fast_path(monkeypatch):
+    """anydoc already reads the text layer in milliseconds and keeps structure."""
+    pytest.importorskip("anydoc")
+    from application.core.settings import settings
+    from application.parser.file.bulk import get_default_file_extractor
+
+    monkeypatch.setattr(settings, "DOC_PARSER_ENGINE", "anydoc")
+
+    pdf_parser = get_default_file_extractor(pdf_text_fast_path=True)[".pdf"]
+
+    assert type(pdf_parser).__name__ == "AnydocParser"
+
+
+def test_fast_path_does_not_change_non_pdf_parsers(docling_engine):
     from application.parser.file.bulk import get_default_file_extractor
 
     plain = get_default_file_extractor()

tests/parser/file/test_tableize.py

@@ -0,0 +1,94 @@
+"""Tests for the dot-leader/whitespace table reconstruction (``ANYDOC_TABLEIZE``)."""
+from application.parser.file.tableize import tableize
+
+DOT_LEADER = """Revenues
+Insurance premiums ............ 83,431 77,731
+Sales and service revenues ........ 24,660 23,406
+Freight rail transportation ......... 22,341 23,852
+Total revenues 130,432 124,989
+"""
+
+
+def test_dot_leader_run_becomes_table():
+    out = tableize(DOT_LEADER)
+    assert "| Insurance premiums | 83,431 | 77,731 |" in out
+    assert "| Total revenues | 130,432 | 124,989 |" in out
+    assert "| --- |" in out
+    assert "Revenues" in out  # heading line untouched
+
+
+def test_whitespace_only_rows_convert_too():
+    md = "Alpha 1 2\nBeta 3 4\nGamma 5 6\n"
+    out = tableize(md)
+    assert "| Alpha | 1 | 2 |" in out
+
+
+def test_currency_symbol_merges_with_number():
+    md = "Cash $ 1,234 900\nDebt $ 2,000 1,500\nEquity $ 900 800\n"
+    out = tableize(md)
+    assert "| Cash | $1,234 | 900 |" in out
+
+
+def test_parenthesised_negatives_and_dash():
+    md = "Losses (30) —\nGains (12) —\nNet (42) —\n"
+    out = tableize(md)
+    assert "| Losses | (30) | — |" in out
+
+
+def test_short_run_is_left_alone():
+    md = "Alpha 1 2\nBeta 3 4\n"
+    assert "|" not in tableize(md)
+
+
+def test_mixed_widths_are_left_alone():
+    md = "Alpha 1 2\nBeta 3\nGamma 5 6\n"
+    assert "|" not in tableize(md)
+
+
+def test_prose_with_numbers_is_left_alone():
+    md = "The division shipped 47 releases in 2023.\nIt hired 12 engineers this year alone.\nRevenue grew by a factor of 3 since 2019.\n"
+    assert "|" not in tableize(md)
+
+
+def test_min_rows_boundary():
+    two = "A 1 2\nB 3 4\n"
+    three = two + "C 5 6\n"
+    assert "|" not in tableize(two, min_rows=3)
+    assert "|" in tableize(three, min_rows=3)
+
+
+def test_pipe_in_label_is_escaped():
+    md = "Assets | current 1,234 900\nDebt 2,000 1,500\nEquity 900 800\n"
+    out = tableize(md)
+    assert "| Assets \\| current | 1,234 | 900 |" in out
+
+
+def test_dollar_prefixed_word_is_not_a_value():
+    md = "Alpha $TBD 2\nBeta $TBD 4\nGamma $TBD 6\n"
+    assert "|" not in tableize(md)
+
+
+def test_non_table_text_passes_through_verbatim():
+    """Only the trailing newline may differ (splitlines/join round-trip)."""
+    md = "# Heading\n\nA paragraph with no numbers.\n\n- a list item\n"
+    assert tableize(md) == md.rstrip("\n")
+
+
+def test_single_trailing_number_lines_are_not_a_table():
+    """Headings, footnotes and version lists look like 'word number' rows; leave them alone."""
+    from application.parser.file.tableize import tableize
+
+    for block in (
+        "Chapter 1\nChapter 2\nChapter 3",
+        "Footnote 1\nFootnote 2\nFootnote 3",
+        "ISO 9001\nISO 14001\nISO 27001",
+        "Version 2.0\nVersion 3.0\nVersion 4.0",
+    ):
+        assert tableize(block) == block
+
+
+def test_leader_rows_with_one_value_still_convert():
+    from application.parser.file.tableize import tableize
+
+    block = "Revenue ...... 1,234\nCosts ...... 567\nProfit ...... 667"
+    assert "| --- |" in tableize(block)

tests/parser/remote/test_s3_loader.py

@@ -903,3 +903,14 @@ class TestSSRFValidation:
         with pytest.raises(ValueError, match="Invalid S3 endpoint_url"):
             s3_loader.load_data(input_data)
         mock_boto3.client.assert_not_called()
+
+
+def test_is_supported_document_follows_the_upload_whitelist():
+    from application.parser.file.constants import SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS
+    from application.parser.remote.s3_loader import S3Loader
+
+    for suffix in SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS:
+        key = f"bucket/file{suffix}"
+        assert S3Loader.is_text_file(None, key) or S3Loader.is_supported_document(None, key), suffix
+    assert S3Loader.is_supported_document(None, "bucket/file.doc")
+    assert not S3Loader.is_supported_document(None, "bucket/file.exe")

tests/parser/test_document_reader.py

@@ -50,6 +50,19 @@ def test_zip_bomb_declared_size_over_cap_is_rejected(monkeypatch):
     assert "error" in out and "too much data" in out["error"]
 
 
+@pytest.mark.unit
+@pytest.mark.parametrize(
+    "suffix", [".docm", ".xlsm", ".xlsb", ".pptm", ".ppsx", ".ppsm", ".odt", ".ods", ".odp"]
+)
+def test_zip_bomb_guard_covers_every_anydoc_zip_format(monkeypatch, suffix):
+    """Chat attachments have no extension whitelist, so every zip-packaged format
+    the anydoc map routes must be gated the same way as .docx."""
+    monkeypatch.setattr(dr.settings, "DOCUMENT_MAX_DECOMPRESSED_BYTES", 1000, raising=False)
+    data = _make_zip({"content.xml": b"A" * 50_000})
+    reason = dr._reject_zip_bomb(data, suffix)
+    assert reason is not None and "too much data" in reason
+
+
 @pytest.mark.unit
 def test_zip_too_many_entries_is_rejected(monkeypatch):
     monkeypatch.setattr(dr.settings, "DOCUMENT_MAX_ARCHIVE_ENTRIES", 3, raising=False)
@@ -186,7 +199,7 @@ class _FakeParser:
 @pytest.mark.unit
 def test_engine_picks_parser_and_coerces_list(monkeypatch):
     fake = _FakeParser(["chunk A", "chunk B"])
-    monkeypatch.setattr(dr, "get_default_file_extractor", lambda ocr_enabled=None: {".pdf": fake})
+    monkeypatch.setattr(dr, "get_default_file_extractor", lambda ocr_enabled=None, **kwargs: {".pdf": fake})
     out = parse_document_bytes(b"%PDF-1.4", "doc.pdf", output="text", engine="docling", include_tables=False)
     assert out["content"] == "chunk A\n\nchunk B"
 
@@ -213,6 +226,9 @@ def test_default_markdown_with_tables_converts_docling_once(monkeypatch):
     import sys
     import types
 
+    # engine=auto resolves to the server default; this test is about the docling path.
+    monkeypatch.setattr(dr.settings, "DOC_PARSER_ENGINE", "docling")
+
     from application.parser.file.docling_parser import DoclingParser
 
     counter = {"instances": 0, "converts": 0}
@@ -272,7 +288,7 @@ def test_default_markdown_with_tables_converts_docling_once(monkeypatch):
             # Only reached if the collapse path fails; the test asserts it doesn't.
             return "# parse-file content"
 
-    monkeypatch.setattr(dr, "get_default_file_extractor", lambda ocr_enabled=None: {".pdf": _FakeDoclingParser()})
+    monkeypatch.setattr(dr, "get_default_file_extractor", lambda ocr_enabled=None, **kwargs: {".pdf": _FakeDoclingParser()})
 
     # Defaults: output=markdown, engine=auto, include_tables=True -> the double-parse path.
     out = parse_document_bytes(b"%PDF-1.4", "doc.pdf")
@@ -293,7 +309,7 @@ def test_default_markdown_with_tables_converts_docling_once(monkeypatch):
 @pytest.mark.unit
 def test_pages_slices_form_feed_blob(monkeypatch):
     fake = _FakeParser("page1\fpage2\fpage3")
-    monkeypatch.setattr(dr, "get_default_file_extractor", lambda ocr_enabled=None: {".pdf": fake})
+    monkeypatch.setattr(dr, "get_default_file_extractor", lambda ocr_enabled=None, **kwargs: {".pdf": fake})
     out = parse_document_bytes(b"%PDF", "doc.pdf", output="text", pages="2", engine="docling", include_tables=False)
     assert out["content"] == "page2"
 
@@ -443,7 +459,7 @@ def test_temp_file_cleaned_up_on_parser_error(monkeypatch):
 
     fake = _FakeParser("x")
     fake.parse_file = lambda *a, **k: (_ for _ in ()).throw(RuntimeError("boom"))
-    monkeypatch.setattr(dr, "get_default_file_extractor", lambda ocr_enabled=None: {".pdf": fake})
+    monkeypatch.setattr(dr, "get_default_file_extractor", lambda ocr_enabled=None, **kwargs: {".pdf": fake})
 
     out = parse_document_bytes(b"%PDF", "doc.pdf", output="text", engine="docling", include_tables=False)
     assert "error" in out and "parsing failed" in out["error"]
@@ -455,11 +471,11 @@ def test_temp_file_cleaned_up_on_parser_error(monkeypatch):
 # ---------------------------------------------------------------------------
 @pytest.mark.unit
 def test_ocr_resolution(monkeypatch):
-    monkeypatch.setattr(dr.settings, "DOCLING_OCR_ENABLED", True, raising=False)
+    monkeypatch.setattr(dr.settings, "OCR_ENABLED", True, raising=False)
     assert dr._resolve_ocr_enabled("off") is False
     assert dr._resolve_ocr_enabled("on") is True
     assert dr._resolve_ocr_enabled("auto") is True
-    monkeypatch.setattr(dr.settings, "DOCLING_OCR_ENABLED", False, raising=False)
+    monkeypatch.setattr(dr.settings, "OCR_ENABLED", False, raising=False)
     assert dr._resolve_ocr_enabled("auto") is False
 
 
@@ -622,3 +638,183 @@ def test_structured_tables_render_blank_cells_as_strings(monkeypatch, tmp_path):
     assert all(isinstance(c, str) for c in cells), f"non-string cells: {cells!r}"
     assert "1001" in cells and "1001.0" not in cells
     assert "" in cells  # blank cell renders as empty string, not "nan"/"None"
+
+
+# ---------------------------------------------------------------------------
+# engine=anydoc / auto: the default engine must never trigger a docling pass
+# ---------------------------------------------------------------------------
+@pytest.mark.unit
+def test_anydoc_engine_is_accepted_and_passed_through(monkeypatch):
+    fake = _FakeParser("anydoc text")
+    seen = {}
+
+    def _extractor(ocr_enabled=None, **kwargs):
+        seen.update(kwargs)
+        return {".pdf": fake}
+
+    monkeypatch.setattr(dr, "get_default_file_extractor", _extractor)
+    out = parse_document_bytes(b"%PDF-1.4", "doc.pdf", output="text", engine="anydoc", include_tables=False)
+    assert out["content"] == "anydoc text"
+    assert seen == {"engine": "anydoc"}
+
+
+@pytest.mark.unit
+def test_auto_engine_under_anydoc_skips_docling_table_pass(monkeypatch):
+    """Default read (markdown, engine=auto, include_tables) must not run a docling conversion
+    just to harvest tables when the configured engine is anydoc — that pass would cost more
+    than the whole read and re-import the stack anydoc exists to avoid."""
+    monkeypatch.setattr(dr.settings, "DOC_PARSER_ENGINE", "anydoc")
+    fake = _FakeParser("# md body")
+    monkeypatch.setattr(dr, "get_default_file_extractor", lambda ocr_enabled=None, **kwargs: {".pdf": fake})
+
+    def _boom(*args, **kwargs):
+        raise AssertionError("docling conversion attempted under the anydoc engine")
+
+    monkeypatch.setattr(dr, "_docling_structured", _boom)
+
+    out = parse_document_bytes(b"%PDF-1.4", "doc.pdf")  # markdown, auto, include_tables=True
+
+    assert out["content"] == "# md body"
+    assert "tables" not in out
+
+
+@pytest.mark.unit
+def test_explicit_docling_engine_still_collects_tables(monkeypatch):
+    monkeypatch.setattr(dr.settings, "DOC_PARSER_ENGINE", "anydoc")
+    fake = _FakeParser("body")
+    monkeypatch.setattr(dr, "get_default_file_extractor", lambda ocr_enabled=None, **kwargs: {".pdf": fake})
+    monkeypatch.setattr(
+        dr,
+        "_docling_structured",
+        lambda *a, **k: {"markdown": "body", "tables": [{"markdown": "| t |"}], "structured": {}, "page_count": 1},
+    )
+
+    out = parse_document_bytes(b"%PDF-1.4", "doc.pdf", engine="docling")
+
+    assert out["tables"] == [{"markdown": "| t |"}]
+
+
+@pytest.mark.unit
+def test_structured_output_stays_docling_under_anydoc(monkeypatch):
+    monkeypatch.setattr(dr.settings, "DOC_PARSER_ENGINE", "anydoc")
+    calls = []
+
+    def _fake_structured(path, *, ocr_enabled, include_tables, parser=None):
+        calls.append(path)
+        return {"markdown": "# s", "tables": [], "structured": {"texts": []}, "page_count": 1}
+
+    monkeypatch.setattr(dr, "_docling_structured", _fake_structured)
+    out = parse_document_bytes(b"%PDF-1.4", "doc.pdf", output="structured")
+    assert out["output"] == "structured"
+    assert len(calls) == 1
+
+
+# --- second-pass routing fixes ---------------------------------------------------
+
+
+def test_fast_engine_covers_the_whole_legacy_map():
+    """`fast` must never fall through to the configured (anydoc/docling) map."""
+    for suffix, name in {
+        ".xhtml": "HTMLParser",
+        ".pptx": "PPTXParser",
+        ".epub": "EpubParser",
+        ".rst": "RstParser",
+        ".json": "JSONParser",
+    }.items():
+        assert type(dr._legacy_parser_for(suffix)).__name__ == name
+
+
+@pytest.mark.parametrize("suffix", [".odt", ".doc", ".rtf", ".xls", ".ods", ".odp"])
+def test_fast_engine_is_terminal_for_anydoc_only_suffixes(monkeypatch, suffix):
+    """A suffix only anydoc reads has no legacy parser; `fast` must report that
+    rather than silently running the configured engine the caller opted out of."""
+
+    def _boom(*args, **kwargs):
+        raise AssertionError("fast fell through to the configured engine map")
+
+    monkeypatch.setattr(dr, "get_default_file_extractor", _boom)
+    monkeypatch.setattr(dr, "_zip_bomb_reason", lambda *a, **k: None)
+
+    assert dr._pick_parser(suffix, ocr_enabled=False, engine="fast") is None
+    out = parse_document_bytes(b"\xd0\xcf\x11\xe0 bytes", f"doc{suffix}", output="text", engine="fast")
+    assert "fast engine" in out["error"] and suffix in out["error"]
+
+
+def test_fast_engine_still_falls_through_for_images_and_audio(monkeypatch):
+    """Formats with no legacy parser and no anydoc alternative keep the configured map."""
+    seen = []
+    monkeypatch.setattr(dr, "get_default_file_extractor", lambda **kwargs: seen.append(kwargs) or {})
+    for suffix in (".png", ".jpg", ".mp3", ".txt"):
+        dr._pick_parser(suffix, ocr_enabled=True, engine="fast")
+    assert len(seen) == 4
+
+
+def test_tables_are_not_collected_when_the_pdf_parser_is_not_docling(monkeypatch):
+    """Under OCR_BACKEND=native the docling engine hands PDFs to the native OCR
+    parser; a vanilla DocumentConverter table pass would OCR the scan again."""
+
+    from application.parser.file.ocr_parser import NativeOcrPdfParser
+
+    native = NativeOcrPdfParser()
+    native._parser_config = {}
+    monkeypatch.setattr(native, "parse_file", lambda path, errors="ignore": "native text")
+
+    calls = []
+    monkeypatch.setattr(dr, "_pick_parser", lambda *a, **k: native)
+    monkeypatch.setattr(dr, "_effective_engine", lambda engine: "docling")
+    monkeypatch.setattr(dr, "_docling_structured", lambda *a, **k: calls.append(1) or {"tables": []})
+    monkeypatch.setattr(dr, "_zip_bomb_reason", lambda *a, **k: None)
+
+    out = parse_document_bytes(b"%PDF-1.4", "scan.pdf", engine="docling", include_tables=True)
+
+    assert out["content"] == "native text"
+    assert calls == []
+
+
+def test_tables_survive_native_ocr_delegation_for_text_only_pdfs(monkeypatch):
+    """OCR_BACKEND=native under the docling engine wraps a DoclingParser as the
+    native parser's text_parser. A PDF with a text layer on every page is handed
+    to that parser, so tables must ride that single conversion instead of being
+    dropped by the scan exclusion."""
+
+    from application.parser.file.docling_parser import DoclingParser
+    from application.parser.file.ocr_parser import NativeOcrPdfParser
+
+    class _Docling(DoclingParser):
+        def __init__(self):
+            super().__init__()
+            self._parser_config = {"ready": True}
+
+    docling = _Docling()
+    native = NativeOcrPdfParser(text_parser=docling)
+    native._parser_config = {}
+    monkeypatch.setattr(native, "text_layer_delegate", lambda path: docling)
+    monkeypatch.setattr(
+        native, "parse_file", lambda path, errors="ignore": pytest.fail("native OCR ran on a text-only PDF")
+    )
+
+    calls = []
+
+    def _fake_structured(path, *, ocr_enabled, include_tables, parser=None):
+        calls.append(parser)
+        return {"markdown": "# text pdf", "tables": [{"markdown": "| t |"}], "structured": {}, "page_count": 1}
+
+    monkeypatch.setattr(dr, "_pick_parser", lambda *a, **k: native)
+    monkeypatch.setattr(dr, "_effective_engine", lambda engine: "docling")
+    monkeypatch.setattr(dr, "_docling_structured", _fake_structured)
+    monkeypatch.setattr(dr, "_zip_bomb_reason", lambda *a, **k: None)
+
+    out = parse_document_bytes(b"%PDF-1.4", "text.pdf", engine="docling", include_tables=True)
+
+    assert out["content"] == "# text pdf"
+    assert out["tables"] == [{"markdown": "| t |"}]
+    assert calls == [docling]
+
+
+def test_binary_office_suffix_without_anydoc_is_an_error_not_text(monkeypatch):
+    monkeypatch.setattr(dr, "_pick_parser", lambda *a, **k: None)
+    monkeypatch.setattr(dr, "_zip_bomb_reason", lambda *a, **k: None)
+
+    out = parse_document_bytes(b"\xd0\xcf\x11\xe0 OLE bytes", "legacy.doc", output="text")
+
+    assert "firecrawl-anydoc" in out["error"]

tests/requirements.txt

@@ -12,3 +12,6 @@ jupyter-client>=8.0
 # (python-pptx and openpyxl already ship in application/requirements.txt).
 python-docx>=1.1
 reportlab>=4.0,<5
+# The optional docling engine stays installed in the test env so its parser
+# paths keep CI coverage even though production installs it on demand.
+-r ../application/requirements-docling.txt