ocr update

application/Dockerfile

@@ -66,6 +66,16 @@ RUN apt-get update && \
     ln -s /usr/bin/python3.12 /usr/bin/python && \
     rm -rf /var/lib/apt/lists/*
 
+# The recommended OCR engine (OCR_ENGINE=tesseract) shells out to the system
+# tesseract binary, so ship it alongside the optional docling engine. Extra
+# language packs are a deployment concern (apt: tesseract-ocr-<lang>).
+ARG INSTALL_DOCLING=false
+RUN if [ "$INSTALL_DOCLING" = "true" ]; then \
+        apt-get update && \
+        apt-get install -y --no-install-recommends tesseract-ocr tesseract-ocr-eng && \
+        rm -rf /var/lib/apt/lists/*; \
+    fi
+
 # Set working directory
 WORKDIR /app
 

application/api/user/sources/upload.py

@@ -272,7 +272,21 @@ class UploadFile(Resource):
                     # Office/e-book containers are ZIP-based formats but must
                     # be parsed as documents, not expanded as user archives.
                     is_office_format = safe_file.lower().endswith(
-                        (".docx", ".xlsx", ".pptx", ".odt", ".ods", ".odp", ".epub")
+                        (
+                            ".docx",
+                            ".docm",
+                            ".xlsx",
+                            ".xlsm",
+                            ".xlsb",
+                            ".pptx",
+                            ".pptm",
+                            ".ppsx",
+                            ".ppsm",
+                            ".odt",
+                            ".ods",
+                            ".odp",
+                            ".epub",
+                        )
                     )
                     if zipfile.is_zipfile(temp_file_path) and not is_office_format:
                         extract_dir = os.path.join(upload_dir, "extracted")

application/core/settings.py

@@ -129,6 +129,40 @@ class Settings(BaseSettings):
     # and the upload cap is 100 MB, so the gate is what keeps one upload from
     # taking the ingest worker down. 0 disables it.
     MARKUP_MAX_BYTES: int = 8_000_000
+    # Trust-check anydoc's PDF output (application/parser/file/pdf_trust.py):
+    # flag composite (Type0) fonts without a ToUnicode map, and CJK-declaring
+    # PDFs whose extracted text has almost no CJK — the two classes where
+    # anydoc drops text silently. A flagged file re-parses on the docling
+    # fallback when docling is installed; otherwise the anydoc output is kept
+    # and the document gets extra_info["parse_warnings"]. ~30 ms per scanned MB.
+    PDF_TRUST_CHECK: bool = True
+    # Rewrite dot-leader / whitespace-aligned table runs in anydoc's PDF
+    # markdown into GFM tables (application/parser/file/tableize.py). Off by
+    # default: it rewrites content on a heuristic (>=3 uniform label+numbers
+    # lines) validated only on a small corpus so far.
+    ANYDOC_TABLEIZE: bool = False
+    # OCR engine used by the docling parsers when OCR is on
+    # (DOCLING_OCR_ENABLED / DOCLING_OCR_ATTACHMENTS_ENABLED). Benched 2026-08
+    # on EN/ZH/table/degraded scans (docs/Guides/ocr has the menu):
+    #   tesseract — recommended: best classic-engine accuracy (perfect EN word
+    #     recall, 0.000 bilingual CER, 100% table cells), ~35 MB, CPU-only.
+    #     Needs the system binary + language packs (the Docker image installs
+    #     them when built with INSTALL_DOCLING=true).
+    #   auto — docling's pick: ocrmac on macOS (excellent), rapidocr on Linux
+    #     (silently shreds some long text lines — avoid as a server default).
+    #   ocrmac | rapidocr — force one of those.
+    #   deepseek — DeepSeek-OCR through docling's VLM pipeline against an
+    #     Ollama/vLLM endpoint (OCR_DEEPSEEK_*). Best table/CJK quality; the
+    #     worker stays light (~370 MB, no layout models) but each page costs
+    #     seconds on the model server.
+    # An engine that is not installed degrades to "auto" with a warning
+    # instead of failing the parse.
+    OCR_ENGINE: str = "tesseract"
+    # Tesseract language packs, "+"-separated (e.g. "eng+chi_sim+deu"). Other
+    # engines keep their own defaults — their language codes differ.
+    OCR_LANGS: str = "eng"
+    OCR_DEEPSEEK_URL: str = "http://localhost:11434/v1/chat/completions"
+    OCR_DEEPSEEK_MODEL: str = "deepseek-ocr:3b"
     # Chars-per-page floor below which an OCR'd PDF/image parse is treated as a docling
     # pipeline dropout (long-running workers were observed returning zero characters for
     # every scanned page after a long scanned PDF, with no error) rather than as content.

application/parser/file/anydoc_parser.py

@@ -18,6 +18,7 @@ import logging
 from pathlib import Path
 from typing import Dict, List, Optional, Tuple, Union
 
+from application.core.settings import settings
 from application.parser.file.base_parser import (
     BaseParser,
     DocumentParseError,
@@ -27,6 +28,11 @@ from application.parser.file.base_parser import (
 
 logger = logging.getLogger(__name__)
 
+# A fallback parse (scanned-PDF delegation or trust-check re-parse) yielding
+# fewer stripped characters than this is an empty document in the making,
+# not content.
+_MIN_SCAN_FALLBACK_CHARS = 50
+
 # Suffixes anydoc 0.2.3 converts, verified with ``anydoc.format_from_extension``.
 # ``.epub`` is deliberately absent: it stays on ``EpubParser``.
 ANYDOC_SUFFIXES: Tuple[str, ...] = (
@@ -56,11 +62,39 @@ ANYDOC_SUFFIXES: Tuple[str, ...] = (
 )
 
 
+# The five formats every engine has its own parser for (docling / legacy);
+# under the anydoc engine they get that parser as the fallback. Everything
+# else in ``ANYDOC_SUFFIXES`` is anydoc-only ("gained") and is mapped to a
+# fallback-less ``AnydocParser`` in every parser map.
+_CORE_SUFFIXES = frozenset({".pdf", ".docx", ".pptx", ".xlsx", ".csv"})
+ANYDOC_GAINED_SUFFIXES: Tuple[str, ...] = tuple(
+    suffix for suffix in ANYDOC_SUFFIXES if suffix not in _CORE_SUFFIXES
+)
+
+
 def anydoc_available() -> bool:
     """Whether the ``anydoc`` extension can be imported."""
     return module_available("anydoc")
 
 
+def _result_chars(result: Union[str, List[str]]) -> int:
+    """Stripped character count of a parser result (str or list of rows)."""
+    if isinstance(result, list):
+        return sum(len(str(part).strip()) for part in result)
+    return len(str(result).strip())
+
+
+def _is_docling_backed(parser: Optional[BaseParser]) -> bool:
+    """True when ``parser`` is a docling parser (worth a trust-check re-parse)."""
+    if parser is None:
+        return False
+    try:
+        from application.parser.file.docling_parser import DoclingParser
+    except ImportError:
+        return False
+    return isinstance(parser, DoclingParser)
+
+
 class AnydocParser(BaseParser):
     """Markdown conversion via ``anydoc.to_markdown`` with a typed fallback.
 
@@ -82,6 +116,10 @@ class AnydocParser(BaseParser):
         super().__init__(parser_config)
         self.fallback_parser = fallback_parser
         self.last_engine: Optional[str] = None
+        # (path, problems) from the most recent parse whose output the trust
+        # check flagged but kept — surfaced via ``get_file_metadata`` as
+        # ``parse_warnings`` so the document carries its own caveat.
+        self._last_warnings: Optional[Tuple[Path, List[str]]] = None
 
     def _init_parser(self) -> Dict:
         # Import for real rather than trusting ``find_spec``: a wheel whose
@@ -117,6 +155,7 @@ class AnydocParser(BaseParser):
         import anydoc
 
         path = Path(file)
+        self._last_warnings = None
         try:
             content = anydoc.to_markdown(str(path))
         except anydoc.ResourceLimitError as exc:
@@ -132,7 +171,10 @@ class AnydocParser(BaseParser):
             # Typed, per-document failures another engine may still handle:
             # UnsupportedError (scanned PDF / unknown format), MalformedError,
             # EncryptedError, MissingPartError.
-            return self._delegate(path, errors, f"{type(exc).__name__}: {exc}")
+            ocr_needed = isinstance(exc, anydoc.UnsupportedError) and "OCR" in str(exc)
+            return self._delegate(
+                path, errors, f"{type(exc).__name__}: {exc}", ocr_needed=ocr_needed
+            )
         except OSError as exc:
             raise DocumentParseError(
                 f"Failed to parse {path.name}: the file could not be read."
@@ -147,10 +189,117 @@ class AnydocParser(BaseParser):
             return self._delegate(path, errors, "anydoc produced no text")
 
         self.last_engine = "anydoc"
+        if path.suffix.lower() == ".pdf":
+            content = self._finish_pdf(path, content, errors)
         return content
 
-    def _delegate(self, path: Path, errors: str, reason: str) -> Union[str, List[str]]:
-        """Hand ``path`` to the fallback parser, or fail loudly without one."""
+    def _finish_pdf(self, path: Path, content: str, errors: str) -> Union[str, List[str]]:
+        """Post-process a successful anydoc PDF conversion.
+
+        Two PDF-only steps:
+
+        1. Trust check (``PDF_TRUST_CHECK``): the two classes where anydoc
+           drops text *silently* — Type0 fonts without ToUnicode, and
+           CJK-declaring PDFs with CJK-less output. A flagged file re-parses
+           on the docling fallback when one is wired; otherwise the anydoc
+           output is kept and the problems ride along as ``parse_warnings``
+           metadata.
+        2. ``ANYDOC_TABLEIZE`` (off by default): rewrite dot-leader /
+           whitespace table runs as GFM tables. Never applied to a docling
+           re-parse — docling emits real tables already.
+        """
+        problems = self._trust_problems(path, content)
+        if problems:
+            rerouted = self._reroute_flagged(path, errors, problems)
+            if rerouted is not None:
+                return rerouted
+            self._last_warnings = (path, problems)
+            logger.warning(
+                f"anydoc output for {path.name} failed the PDF trust check "
+                f"({'; '.join(problems)}); keeping it with parse_warnings"
+            )
+        if settings.ANYDOC_TABLEIZE:
+            from application.parser.file.tableize import tableize
+
+            content = tableize(content)
+        return content
+
+    def _trust_problems(self, path: Path, content: str) -> List[str]:
+        """Trust-check findings for ``content``; [] when disabled, clean, or the check errors."""
+        if not settings.PDF_TRUST_CHECK:
+            return []
+        try:
+            from application.parser.file.pdf_trust import verify_pdf_file
+
+            return verify_pdf_file(path, content)
+        except Exception:
+            # The check exists to catch silent loss; it must never turn a
+            # successful parse into a failure.
+            logger.warning(
+                f"PDF trust check errored on {path.name}; trusting the output",
+                exc_info=True,
+            )
+            return []
+
+    def _reroute_flagged(
+        self, path: Path, errors: str, problems: List[str]
+    ) -> Optional[Union[str, List[str]]]:
+        """Re-parse a trust-flagged PDF with a docling-backed fallback.
+
+        Returns the fallback's output, or None when there is no docling
+        fallback, it fails, or it comes back near-empty — the caller then
+        keeps the anydoc output.
+        Only docling is worth the re-parse: it ships Adobe's predefined
+        CMaps, which is exactly what the flagged font class needs; the
+        legacy pypdf parser does not.
+        """
+        fallback = self.fallback_parser
+        if not _is_docling_backed(fallback):
+            return None
+        logger.warning(
+            f"anydoc output for {path.name} failed the PDF trust check "
+            f"({'; '.join(problems)}); re-parsing with {type(fallback).__name__}"
+        )
+        try:
+            result = delegate_parse(fallback, path, errors)
+        except DocumentParseError:
+            logger.warning(
+                f"docling re-parse of trust-flagged {path.name} failed; "
+                "keeping the anydoc output",
+                exc_info=True,
+            )
+            return None
+        if _result_chars(result) < _MIN_SCAN_FALLBACK_CHARS:
+            # A docling pipeline dropout ('' / '<!-- image -->') returns
+            # without raising; adopting it would swap anydoc's real text for
+            # an empty document.
+            logger.warning(
+                f"docling re-parse of trust-flagged {path.name} returned almost "
+                "no text; keeping the anydoc output"
+            )
+            return None
+        self.last_engine = getattr(fallback, "last_engine", None) or type(fallback).__name__
+        return result
+
+    def get_file_metadata(self, file: Path) -> Dict:
+        """Surface trust-check findings for the file just parsed, if any."""
+        last = self._last_warnings
+        if last is not None and last[0] == Path(file):
+            return {"parse_warnings": list(last[1])}
+        return {}
+
+    def _delegate(
+        self, path: Path, errors: str, reason: str, ocr_needed: bool = False
+    ) -> Union[str, List[str]]:
+        """Hand ``path`` to the fallback parser, or fail loudly without one.
+
+        ``ocr_needed`` marks anydoc's scanned-PDF refusal ("OCR is required").
+        The fallback still runs first — docling extracts text layers anydoc
+        refuses (seen on degenerate CJK text layers) even with OCR off — but
+        when it comes back near-empty the parse fails loudly, telling the
+        user how to get OCR, instead of silently storing an empty document
+        for a scan.
+        """
         fallback = self.fallback_parser
         if fallback is None:
             self.last_engine = None
@@ -160,5 +309,19 @@ class AnydocParser(BaseParser):
             f"falling back to {type(fallback).__name__}"
         )
         result = delegate_parse(fallback, path, errors)
+        if ocr_needed and _result_chars(result) < _MIN_SCAN_FALLBACK_CHARS:
+            self.last_engine = None
+            if getattr(fallback, "ocr_enabled", False):
+                hint = " even with OCR enabled."
+            else:
+                hint = (
+                    ". Enable OCR to ingest scans: set DOCLING_OCR_ENABLED=true "
+                    "(and DOCLING_OCR_ATTACHMENTS_ENABLED for attachments) with "
+                    "the docling extra installed."
+                )
+            raise DocumentParseError(
+                f"{path.name} appears to be a scanned PDF (no text layer), and "
+                f"{type(fallback).__name__} extracted almost nothing{hint}"
+            )
         self.last_engine = getattr(fallback, "last_engine", None) or type(fallback).__name__
         return result

application/parser/file/bulk.py

@@ -49,6 +49,25 @@ def _wrap_pdf_fast_path(pdf_parser: BaseParser) -> BaseParser:
     )
 
 
+def _gained_format_entries() -> Dict[str, BaseParser]:
+    """Anydoc-only formats (legacy/macro Office, OpenDocument, RTF) for every map.
+
+    anydoc is a core dependency, so these suffixes are parseable under both
+    engines. When anydoc is somehow missing the entries are omitted, and such
+    files fall to ``SimpleDirectoryReader``'s plain-text read — the
+    pre-existing behaviour for unmapped suffixes.
+    """
+    from application.parser.file.anydoc_parser import (
+        ANYDOC_GAINED_SUFFIXES,
+        AnydocParser,
+        anydoc_available,
+    )
+
+    if not anydoc_available():
+        return {}
+    return {suffix: AnydocParser() for suffix in ANYDOC_GAINED_SUFFIXES}
+
+
 def _legacy_file_extractor(pdf_text_fast_path: bool = False) -> Dict[str, BaseParser]:
     """Parser map that needs neither docling nor anydoc.
 
@@ -79,6 +98,7 @@ def _legacy_file_extractor(pdf_text_fast_path: bool = False) -> Dict[str, BasePa
         ".jpg": ImageParser(),
         ".jpeg": ImageParser(),
         **_build_audio_parser_mapping(),
+        **_gained_format_entries(),
     }
 
 
@@ -165,6 +185,8 @@ def _docling_file_extractor(
         ".xml": DoclingXMLParser(),
         # Formats docling doesn't support - use standard parsers
         ".epub": EpubParser(),
+        # Formats only anydoc reads (legacy/macro Office, OpenDocument, RTF)
+        **_gained_format_entries(),
     }
 
 
@@ -211,7 +233,13 @@ def _anydoc_file_extractor(ocr_enabled: bool, pdf_text_fast_path: bool = False)
     )
     extractor = dict(base)
     for suffix in ANYDOC_SUFFIXES:
-        extractor[suffix] = AnydocParser(fallback_parser=base.get(suffix))
+        fallback = base.get(suffix)
+        if isinstance(fallback, AnydocParser):
+            # A gained-format entry from the base map is already a
+            # fallback-less AnydocParser — anydoc delegating to anydoc would
+            # just repeat the same failure.
+            continue
+        extractor[suffix] = AnydocParser(fallback_parser=fallback)
     extractor[".html"] = HTMLMarkdownParser()
     extractor[".xhtml"] = HTMLMarkdownParser()
     return extractor

application/parser/file/constants.py

@@ -1,5 +1,6 @@
 """Shared file-extension constants for parsing and ingestion flows."""
 
+from application.parser.file.anydoc_parser import ANYDOC_GAINED_SUFFIXES
 from application.stt.constants import SUPPORTED_AUDIO_EXTENSIONS
 
 
@@ -16,6 +17,12 @@ SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS = (
     ".json",
     ".xlsx",
     ".pptx",
+    # Read by the HTML parsers on every engine.
+    ".xhtml",
+    # Read by the anydoc engine (legacy/macro Office, OpenDocument, RTF).
+    # Parseable regardless of DOC_PARSER_ENGINE: anydoc is a core dependency,
+    # and both parser maps route these suffixes to it.
+    *ANYDOC_GAINED_SUFFIXES,
 )
 
 SUPPORTED_SOURCE_IMAGE_EXTENSIONS = (".png", ".jpg", ".jpeg")

application/parser/file/docling_parser.py

@@ -10,6 +10,8 @@ import importlib.util
 import logging
 import os
 import re
+import shutil
+import sys
 import tempfile
 import zipfile
 from pathlib import Path
@@ -71,6 +73,112 @@ def _apply_inference_settings() -> None:
         inference.compile_torch_models = settings.DOCLING_COMPILE_TORCH_MODELS
 
 
+_VALID_OCR_ENGINES = ("tesseract", "auto", "ocrmac", "rapidocr", "deepseek")
+
+
+def _resolve_ocr_engine(requested: Optional[str]) -> str:
+    """Resolve the OCR engine to build, degrading to ``auto`` when unavailable.
+
+    ``auto`` is docling's own selection (ocrmac on macOS, rapidocr on a
+    typical Linux server). The degradation is deliberate: OCR is switched on
+    by deployments that expect scans to work, so a missing engine must warn
+    and OCR with what exists rather than fail every parse.
+
+    Args:
+        requested: Engine name, or None to read ``settings.OCR_ENGINE``.
+
+    Returns:
+        One of ``_VALID_OCR_ENGINES``, guaranteed buildable here.
+    """
+    from application.core.settings import settings
+    from application.parser.file.base_parser import module_available
+
+    engine = str(requested or settings.OCR_ENGINE or "auto").strip().lower()
+    if engine not in _VALID_OCR_ENGINES:
+        logger.warning(
+            f"Unknown OCR_ENGINE {engine!r}; using docling auto-selection"
+        )
+        return "auto"
+    if engine == "tesseract" and shutil.which("tesseract") is None:
+        logger.warning(
+            "OCR_ENGINE=tesseract but no tesseract binary is on PATH (install "
+            "tesseract-ocr plus language packs, or build the Docker image with "
+            "INSTALL_DOCLING=true); using docling auto-selection"
+        )
+        return "auto"
+    if engine == "ocrmac" and (
+        sys.platform != "darwin" or not module_available("ocrmac")
+    ):
+        logger.warning(
+            "OCR_ENGINE=ocrmac needs macOS with the ocrmac package; "
+            "using docling auto-selection"
+        )
+        return "auto"
+    if engine == "rapidocr" and not module_available("rapidocr"):
+        logger.warning(
+            "OCR_ENGINE=rapidocr but rapidocr is not installed; "
+            "using docling auto-selection"
+        )
+        return "auto"
+    return engine
+
+
+def _build_ocr_options(
+    engine: str, languages: Optional[List[str]], force_full_page_ocr: bool
+):
+    """docling OCR options for a resolved classic engine.
+
+    Returns None for ``auto`` (docling's default pipeline options already run
+    auto-selection) and on any build failure — the parse then proceeds on the
+    default engine rather than failing; the caller re-applies
+    ``force_full_page_ocr`` onto whatever options end up active.
+
+    Args:
+        engine: A ``_resolve_ocr_engine`` result other than ``deepseek``.
+        languages: Engine-specific language list; None uses the engine's
+            default (tesseract reads ``settings.OCR_LANGS``).
+        force_full_page_ocr: OCR whole pages instead of only bitmap regions.
+    """
+    if engine == "auto":
+        return None
+    from application.core.settings import settings
+
+    try:
+        if engine == "tesseract":
+            from docling.datamodel.pipeline_options import TesseractCliOcrOptions
+
+            langs = (
+                languages
+                or [lang.strip() for lang in settings.OCR_LANGS.split("+") if lang.strip()]
+                or ["eng"]
+            )
+            return TesseractCliOcrOptions(
+                lang=langs, force_full_page_ocr=force_full_page_ocr
+            )
+        if engine == "rapidocr":
+            from docling.datamodel.pipeline_options import RapidOcrOptions
+
+            return RapidOcrOptions(
+                lang=languages or ["english"],
+                force_full_page_ocr=force_full_page_ocr,
+            )
+        if engine == "ocrmac":
+            from docling.datamodel.pipeline_options import OcrMacOptions
+
+            if languages:
+                return OcrMacOptions(
+                    lang=languages, force_full_page_ocr=force_full_page_ocr
+                )
+            return OcrMacOptions(force_full_page_ocr=force_full_page_ocr)
+    except ImportError as e:
+        logger.warning(f"Failed to build {engine} OCR options: {e}")
+        return None
+    except Exception as e:
+        logger.error(f"Error building {engine} OCR options: {e}")
+        return None
+    return None
+
+
 def _tabular_content_size(file: Path) -> int:
     """Effective content size of a tabular file, in bytes.
 
@@ -303,7 +411,7 @@ class DoclingParser(BaseParser):
     - Advanced PDF layout analysis
     - Table structure recognition
     - Reading order detection
-    - OCR for scanned documents (supports RapidOCR)
+    - OCR for scanned documents (engine chosen by ``OCR_ENGINE``)
     - Unified DoclingDocument format
     - Export to Markdown
 
@@ -317,7 +425,7 @@ class DoclingParser(BaseParser):
         ocr_enabled: bool = True,
         table_structure: bool = True,
         export_format: str = "markdown",
-        use_rapidocr: bool = True,
+        ocr_engine: Optional[str] = None,
         ocr_languages: Optional[List[str]] = None,
         force_full_page_ocr: bool = False,
     ):
@@ -327,29 +435,33 @@ class DoclingParser(BaseParser):
             ocr_enabled: Enable OCR for bitmap/image regions in documents
             table_structure: Enable table structure recognition
             export_format: Output format ('markdown', 'text', 'html')
-            use_rapidocr: Use RapidOCR engine (default True, works well in Docker)
-            ocr_languages: List of OCR languages (default: ['english'])
+            ocr_engine: OCR engine when OCR is enabled — one of
+                ``tesseract | auto | ocrmac | rapidocr | deepseek``. None
+                reads ``settings.OCR_ENGINE`` at converter build time; an
+                unavailable engine degrades to docling's auto-selection with
+                a warning.
+            ocr_languages: Engine-specific language list; None keeps the
+                engine's own default (tesseract reads ``settings.OCR_LANGS``).
             force_full_page_ocr: Force OCR on entire page (False = smart hybrid OCR)
         """
         super().__init__()
         self.ocr_enabled = ocr_enabled
         self.table_structure = table_structure
         self.export_format = export_format
-        self.use_rapidocr = use_rapidocr
-        self.ocr_languages = ocr_languages or ["english"]
+        self.ocr_engine = ocr_engine
+        self.ocr_languages = ocr_languages
         self.force_full_page_ocr = force_full_page_ocr
         self._converter = None
 
     def _create_converter(self):
-        """Create a docling converter with hybrid OCR configuration.
+        """Create a docling converter for the configured OCR engine.
 
-        Uses smart OCR approach:
-        - When ocr_enabled=True and force_full_page_ocr=False (default):
-          Layout model detects text vs bitmap regions, OCR only runs on bitmaps
-        - When ocr_enabled=True and force_full_page_ocr=True:
-          OCR runs on entire page (for scanned documents/images)
-        - When ocr_enabled=False:
-          No OCR, only native text extraction
+        - ``ocr_enabled=False``: no OCR, native text extraction only.
+        - Classic engines (tesseract / auto / ocrmac / rapidocr): the standard
+          PDF pipeline (layout + TableFormer) with that engine's OCR options.
+          ``force_full_page_ocr=False`` (default) OCRs only the bitmap regions
+          the layout model finds; True routes whole pages through OCR.
+        - ``deepseek``: the VLM pipeline instead (``_create_vlm_converter``).
 
         Returns:
             DocumentConverter instance
@@ -364,6 +476,10 @@ class DoclingParser(BaseParser):
 
         _apply_inference_settings()
 
+        engine = _resolve_ocr_engine(self.ocr_engine) if self.ocr_enabled else None
+        if engine == "deepseek":
+            return self._create_vlm_converter()
+
         pipeline_options = PdfPipelineOptions(
             do_ocr=self.ocr_enabled,
             do_table_structure=self.table_structure,
@@ -371,13 +487,15 @@ class DoclingParser(BaseParser):
         _apply_pipeline_caps(pipeline_options)
 
         if self.ocr_enabled:
-            ocr_options = self._get_ocr_options()
+            ocr_options = _build_ocr_options(
+                engine, self.ocr_languages, self.force_full_page_ocr
+            )
             if ocr_options is not None:
                 pipeline_options.ocr_options = ocr_options
             # Docling's *default* OCR options carry their own flag, so without
-            # this the setting was silently dropped whenever `_get_ocr_options`
-            # returned None (use_rapidocr=False) — including the dropout retry,
-            # whose whole point is forcing full-page OCR.
+            # this the setting was silently dropped whenever no explicit
+            # options were built (engine=auto, or a build failure) — including
+            # the dropout retry, whose whole point is forcing full-page OCR.
             active_ocr_options = getattr(pipeline_options, "ocr_options", None)
             if hasattr(active_ocr_options, "force_full_page_ocr"):
                 active_ocr_options.force_full_page_ocr = self.force_full_page_ocr
@@ -393,12 +511,55 @@ class DoclingParser(BaseParser):
             }
         )
 
+    def _create_vlm_converter(self):
+        """DeepSeek-OCR converter via docling's VLM pipeline.
+
+        Each page goes to an OpenAI-compatible endpoint (Ollama or vLLM;
+        ``OCR_DEEPSEEK_URL`` / ``OCR_DEEPSEEK_MODEL``) and the grounded output
+        is parsed back into a DoclingDocument. This replaces the *entire*
+        classic pipeline — no layout/TableFormer/OCR models load in the worker
+        (~370 MB RSS vs 1.0-1.6 GB measured), the compute lives in the model
+        server. Bench trade-offs (2026-08): best table/CJK/degraded-scan
+        quality of every engine tried, ~10-20 s/page on modest hardware, and
+        occasional silent drops of page-level elements (titles).
+        """
+        from docling.datamodel import vlm_model_specs
+        from docling.datamodel.pipeline_options import VlmPipelineOptions
+        from docling.document_converter import (
+            DocumentConverter,
+            ImageFormatOption,
+            InputFormat,
+            PdfFormatOption,
+        )
+        from docling.pipeline.vlm_pipeline import VlmPipeline
+
+        from application.core.settings import settings
+
+        vlm_options = vlm_model_specs.DEEPSEEKOCR_OLLAMA.model_copy(deep=True)
+        vlm_options.url = settings.OCR_DEEPSEEK_URL
+        vlm_options.params["model"] = settings.OCR_DEEPSEEK_MODEL
+        pipeline_options = VlmPipelineOptions(
+            vlm_options=vlm_options, enable_remote_services=True
+        )
+        return DocumentConverter(
+            format_options={
+                InputFormat.PDF: PdfFormatOption(
+                    pipeline_cls=VlmPipeline, pipeline_options=pipeline_options
+                ),
+                InputFormat.IMAGE: ImageFormatOption(
+                    pipeline_cls=VlmPipeline, pipeline_options=pipeline_options
+                ),
+            }
+        )
+
     def _init_parser(self) -> Dict:
         """Initialize the docling converter with hybrid OCR."""
+        from application.core.settings import settings
+
         logger.info("Initializing DoclingParser...")
         logger.info(f"  ocr_enabled={self.ocr_enabled}")
         logger.info(f"  force_full_page_ocr={self.force_full_page_ocr}")
-        logger.info(f"  use_rapidocr={self.use_rapidocr}")
+        logger.info(f"  ocr_engine={self.ocr_engine or settings.OCR_ENGINE}")
 
         if importlib.util.find_spec("docling.document_converter") is None:
             raise ImportError(
@@ -414,34 +575,11 @@ class DoclingParser(BaseParser):
             "ocr_enabled": self.ocr_enabled,
             "table_structure": self.table_structure,
             "export_format": self.export_format,
-            "use_rapidocr": self.use_rapidocr,
+            "ocr_engine": self.ocr_engine,
             "ocr_languages": self.ocr_languages,
             "force_full_page_ocr": self.force_full_page_ocr,
         }
 
-    def _get_ocr_options(self):
-        """Get OCR options based on configuration.
-
-        Returns RapidOcrOptions if use_rapidocr is True and available,
-        otherwise returns None to use docling defaults.
-        """
-        if not self.use_rapidocr:
-            return None
-
-        try:
-            from docling.datamodel.pipeline_options import RapidOcrOptions
-
-            return RapidOcrOptions(
-                lang=self.ocr_languages,
-                force_full_page_ocr=self.force_full_page_ocr,
-            )
-        except ImportError as e:
-            logger.warning(f"Failed to import RapidOcrOptions: {e}")
-            return None
-        except Exception as e:
-            logger.error(f"Error creating RapidOcrOptions: {e}")
-            return None
-
     def _export_content(self, document) -> str:
         """Export document content in the configured format.
 
@@ -651,7 +789,7 @@ class DoclingParser(BaseParser):
 
 
 class DoclingPDFParser(DoclingParser):
-    """Docling-based PDF parser with advanced features and RapidOCR support.
+    """Docling-based PDF parser with advanced features and configurable OCR.
 
     Uses hybrid OCR approach by default:
     - Text regions: Direct PDF text extraction (fast)
@@ -664,7 +802,7 @@ class DoclingPDFParser(DoclingParser):
         self,
         ocr_enabled: bool = True,
         table_structure: bool = True,
-        use_rapidocr: bool = True,
+        ocr_engine: Optional[str] = None,
         ocr_languages: Optional[List[str]] = None,
         force_full_page_ocr: bool = False,
     ):
@@ -672,7 +810,7 @@ class DoclingPDFParser(DoclingParser):
             ocr_enabled=ocr_enabled,
             table_structure=table_structure,
             export_format="markdown",
-            use_rapidocr=use_rapidocr,
+            ocr_engine=ocr_engine,
             ocr_languages=ocr_languages,
             force_full_page_ocr=force_full_page_ocr,
         )
@@ -723,7 +861,7 @@ class DoclingHTMLParser(DoclingParser):
 
 
 class DoclingImageParser(DoclingParser):
-    """Docling-based image parser with OCR and RapidOCR support.
+    """Docling-based image parser with configurable OCR.
 
     For images, force_full_page_ocr=True is used since images are entirely
     visual and require full OCR to extract any text.
@@ -732,14 +870,14 @@ class DoclingImageParser(DoclingParser):
     def __init__(
         self,
         ocr_enabled: bool = True,
-        use_rapidocr: bool = True,
+        ocr_engine: Optional[str] = None,
         ocr_languages: Optional[List[str]] = None,
         force_full_page_ocr: bool = True,
     ):
         super().__init__(
             ocr_enabled=ocr_enabled,
             export_format="markdown",
-            use_rapidocr=use_rapidocr,
+            ocr_engine=ocr_engine,
             ocr_languages=ocr_languages,
             force_full_page_ocr=force_full_page_ocr,
         )

application/parser/file/pdf_trust.py

@@ -0,0 +1,177 @@
+"""Trust checks for PDF text extraction.
+
+anydoc silently drops text from fonts it cannot map to Unicode — seen with
+non-embedded composite (Type0/CID) fonts relying on predefined CMaps, where
+the EN/ZH NDA corpus file loses its whole Chinese column with no error, and
+with Adobe-CNS1 fonts in HK legal PDFs where a valid ToUnicode exists but
+extraction still fails. These dependency-free checks scan the PDF's raw
+objects (including Flate-compressed object streams) so the pipeline can
+route such files to a heavier parser, or at least mark the output as
+unverified, instead of trusting silent partial text.
+
+Two stages:
+
+* ``check_pdf_fonts`` — pre-flight on the bytes alone: composite fonts
+  without an embedded ToUnicode CMap, and whether the PDF declares CJK font
+  resources at all.
+* ``verify_extraction`` — pre-flight plus the cross-check the font scan
+  cannot do: a PDF that declares CJK fonts whose extracted text contains
+  almost no CJK characters is not to be trusted.
+
+A flag means "verify or route to a fallback", not "the text is wrong": tools
+shipping Adobe's predefined CMaps (docling's parser does) often extract such
+files fine. On the benchmark corpus the checks caught both known
+silent-drop cases with zero false positives on the other 14 PDFs, and cost
+~30 ms per scanned MB (92 ms on a 3 MB, 150-page annual report).
+"""
+import re
+import zlib
+from pathlib import Path
+from typing import Dict, Iterator, List, Union
+
+_OBJ = re.compile(rb"\d+\s+\d+\s+obj(.*?)endobj", re.DOTALL)
+_STREAM = re.compile(rb"stream\r?\n(.*?)\r?\nendstream", re.DOTALL)
+_TYPE0 = re.compile(rb"/Subtype\s*/Type0")
+# The CJK expectation is keyed on CIDSystemInfo /Ordering ALONE — the
+# authoritative "this PDF maps text through a CJK character collection"
+# signal, present in every known silent-drop case. Matching CJK font *names*
+# (SimSun, MS-Gothic, MSungHK, ...) was tried and rejected: Word-exported
+# Latin PDFs routinely embed an MS-Gothic subset for a stray full-width
+# character (a 150-page English annual report in the benchmark corpus does),
+# and substring matching also catches Latin faces like FranklinGothic —
+# either way English-only documents would be rerouted to the heavy engine.
+_CJK_ORDERING = re.compile(rb"/Ordering\s*\((GB1|CNS1|Japan1|Japan2|KR|Korea1)\)")
+# Bytes kept around a Type0 marker found inside a decompressed object
+# stream: object streams hold many dicts with no obj/endobj markers, and a
+# font dict's keys sit close to its /Subtype entry.
+_WINDOW_BEFORE, _WINDOW_AFTER = 200, 800
+
+# Extracted text with fewer CJK characters than this, from a PDF that
+# declares CJK fonts, is treated as a silent drop.
+_MIN_CJK_CHARS = 10
+
+_CJK_RANGES = (
+    ("一", "鿿"),  # CJK Unified Ideographs
+    ("぀", "ヿ"),  # Hiragana + Katakana
+    ("가", "힯"),  # Hangul syllables
+)
+
+
+# Per-stream decompression cap. Flate reaches ~1000:1, so uncapped inflation
+# of a crafted (or merely image-heavy) PDF inside the upload cap could balloon
+# to gigabytes and OOM the ingest worker. Font dicts and object streams — the
+# only things the signals live in — are far smaller than this.
+_STREAM_INFLATE_CAP = 4_000_000
+
+
+def _decompressed_streams(raw: bytes) -> Iterator[bytes]:
+    """Each Flate stream in ``raw`` that decompresses, one at a time, capped."""
+    for match in _STREAM.finditer(raw):
+        try:
+            inflated = zlib.decompressobj().decompress(
+                match.group(1), _STREAM_INFLATE_CAP
+            )
+        except zlib.error:
+            continue
+        yield inflated
+
+
+def _cjk_chars(text: str, up_to: int) -> int:
+    """Count CJK characters in ``text``, stopping once ``up_to`` is reached."""
+    count = 0
+    for char in text:
+        for low, high in _CJK_RANGES:
+            if low <= char <= high:
+                count += 1
+                break
+        if count >= up_to:
+            break
+    return count
+
+
+def check_pdf_fonts(data: bytes) -> Dict[str, Union[int, bool]]:
+    """Pre-flight font scan of a PDF's bytes.
+
+    Args:
+        data: The complete PDF file contents.
+
+    Returns:
+        Dict with ``type0`` (composite fonts seen), ``type0_no_tounicode``
+        (those without an embedded ToUnicode CMap), ``expects_cjk`` (the PDF
+        declares CJK font resources), ``has_fonts``, and ``flagged`` — True
+        when extraction should not be trusted unverified.
+    """
+    type0 = bare = 0
+    expects_cjk = bool(_CJK_ORDERING.search(data))
+    has_fonts = b"/Font" in data
+
+    def count_font(unit: bytes) -> None:
+        """One analysis unit holding a Type0 font dict."""
+        nonlocal type0, bare
+        type0 += 1
+        if b"/ToUnicode" not in unit:
+            bare += 1
+
+    for match in _OBJ.finditer(data):
+        body = match.group(1)
+        if _TYPE0.search(body):
+            count_font(body)
+    # Each decompressed stream is scanned for every signal in one pass and
+    # then dropped; nothing inflated is retained past its loop iteration.
+    for stream in _decompressed_streams(data):
+        expects_cjk = expects_cjk or bool(_CJK_ORDERING.search(stream))
+        has_fonts = has_fonts or b"/Font" in stream
+        for type0_match in _TYPE0.finditer(stream):
+            start = type0_match.start()
+            count_font(stream[max(0, start - _WINDOW_BEFORE): start + _WINDOW_AFTER])
+    return {
+        "type0": type0,
+        "type0_no_tounicode": bare,
+        "expects_cjk": expects_cjk,
+        "has_fonts": has_fonts,
+        "flagged": bare > 0 or not has_fonts,
+    }
+
+
+def verify_extraction(data: bytes, markdown: str) -> List[str]:
+    """Reasons the extracted ``markdown`` of the PDF ``data`` shouldn't be trusted.
+
+    Combines the pre-flight font scan with the post-conversion cross-check it
+    cannot do alone: fonts with a valid ToUnicode that the converter
+    nevertheless failed to extract still show up as missing CJK output.
+
+    Args:
+        data: The complete PDF file contents.
+        markdown: The text a converter extracted from it.
+
+    Returns:
+        Human-readable problem strings; empty when the output looks sound.
+    """
+    problems: List[str] = []
+    result = check_pdf_fonts(data)
+    if result["type0_no_tounicode"]:
+        problems.append(
+            f"{result['type0_no_tounicode']} composite (Type0) font(s) carry no "
+            "ToUnicode map; extracted text may silently omit glyphs"
+        )
+    elif not result["has_fonts"]:
+        problems.append(
+            "no font resources detected; text extraction cannot be verified"
+        )
+    if result["expects_cjk"]:
+        cjk = _cjk_chars(markdown, _MIN_CJK_CHARS)
+        if cjk < _MIN_CJK_CHARS:
+            problems.append(
+                f"PDF declares CJK fonts but the extracted text contains only "
+                f"{cjk} CJK character(s)"
+            )
+    return problems
+
+
+def verify_pdf_file(file: Path, markdown: str) -> List[str]:
+    """``verify_extraction`` for a file on disk; unreadable files trust the output."""
+    try:
+        data = Path(file).read_bytes()
+    except OSError:
+        return []
+    return verify_extraction(data, markdown)

application/parser/file/tableize.py

@@ -0,0 +1,94 @@
+"""Reconstruct typographic tables in anydoc's PDF Markdown output.
+
+anydoc converts PDFs straight to Markdown with no document model, so tables
+drawn with dot leaders or bare whitespace alignment (financial statements,
+tables of contents) come out as flat text lines — values intact, structure
+lost. This post-processor detects runs of such lines and rewrites them as
+GFM tables: the lightweight alternative to a table-structure model for this
+layout family. (The robust fix is upstream in anydoc's Rust PDF code, where
+glyph x-positions exist; this is the recoverable-downstream version.)
+
+Deliberately conservative — a run converts only when it has at least
+``min_rows`` consecutive lines that each parse as ``label [leaders]
+numeric-columns`` with the *same* column count; anything else passes through
+untouched. Gated by ``ANYDOC_TABLEIZE`` (off by default) and applied only to
+anydoc's own PDF output, never to docling's.
+"""
+import re
+from typing import List, Optional, Tuple
+
+# label ...... 1,234 (56) — dot-leader row
+_LEADER = re.compile(r"^(.*?)\s*\.{3,}\s*(.+)$")
+# numeric-ish token: $ 1,234 / (30) / 83,431 / 12.5% / —
+_NUM_TOKEN = re.compile(r"\$?\(?-?\d[\d,.]*\)?%?|—")
+# row without leaders: label text, then a trailing run of numeric tokens
+_TRAILING = re.compile(r"^(.*?[^\s\d,.])\s+([\d$(].*)$")
+
+
+def _merge_currency(tokens: List[str]) -> List[str]:
+    """Join a free-standing ``$`` onto the number that follows it."""
+    out: List[str] = []
+    for token in tokens:
+        if out and out[-1] == "$":
+            out[-1] = "$" + token
+        else:
+            out.append(token)
+    return out
+
+
+def _parse_row(line: str) -> Optional[Tuple[str, List[str]]]:
+    """``(label, values)`` when the line looks like a typographic table row, else None."""
+    match = _LEADER.match(line) or _TRAILING.match(line)
+    if not match:
+        return None
+    label, rest = match.group(1).strip(" ."), match.group(2)
+    values = _merge_currency(rest.split())
+    if not values or not all(_NUM_TOKEN.fullmatch(v.lstrip("$")) for v in values):
+        return None
+    if not label or _NUM_TOKEN.fullmatch(label):
+        return None
+    return label, values
+
+
+def tableize(markdown: str, min_rows: int = 3) -> str:
+    """Rewrite runs of typographic table rows in ``markdown`` as GFM tables.
+
+    Args:
+        markdown: Converter output to post-process.
+        min_rows: Minimum consecutive, same-width rows for a run to convert.
+
+    Returns:
+        ``markdown`` with qualifying runs rewritten; everything else verbatim.
+    """
+    out: List[str] = []
+    run: List[Tuple[str, List[str], str]] = []  # (label, values, original line)
+
+    def flush() -> None:
+        nonlocal run
+        widths = {len(values) for _, values, _ in run}
+        if len(run) >= min_rows and len(widths) == 1:
+            ncols = widths.pop()
+            out.append("")
+            out.append("| " + " | ".join([" "] + [f"col{i + 1}" for i in range(ncols)]) + " |")
+            out.append("|" + " --- |" * (ncols + 1))
+            for label, values, _ in run:
+                # Only the label can carry a '|' (values are numeric tokens);
+                # unescaped it would add a cell and mis-column the row.
+                cells = [label.replace("|", "\\|")] + values
+                out.append("| " + " | ".join(cells) + " |")
+            out.append("")
+        else:
+            out.extend(original for _, _, original in run)
+        run = []
+
+    for line in markdown.splitlines():
+        parsed = _parse_row(line.strip()) if line.strip() else None
+        if parsed:
+            run.append((*parsed, line))
+        else:
+            if run:
+                flush()
+            out.append(line)
+    if run:
+        flush()
+    return "\n".join(out)

application/worker.py

@@ -1777,7 +1777,7 @@ def attachment_worker(self, file_info, user):
         parser_metadata = {
             key: value
             for key, value in (attachment_document.extra_info or {}).items()
-            if key.startswith("transcript_")
+            if key.startswith("transcript_") or key == "parse_warnings"
         }
         if parser_metadata:
             metadata = {**metadata, **parser_metadata}