Fixes batch 1

application/core/settings.py

@@ -183,9 +183,10 @@ class Settings(BaseSettings):
     OCR_LANGS: str = "eng"
     OCR_DEEPSEEK_URL: str = "http://localhost:11434/v1/chat/completions"
     OCR_DEEPSEEK_MODEL: str = "deepseek-ocr:3b"
-    # Native backend only: seconds allowed per page request to the DeepSeek
-    # endpoint (pages are sent one at a time). A 3B model on a laptop needs
-    # minutes; a vLLM GPU deployment, seconds.
+    # Seconds allowed per page request to the DeepSeek endpoint, on both
+    # backends (native sends pages one at a time; docling's VLM pipeline
+    # keeps its own concurrency). A 3B model on a laptop needs minutes; a
+    # vLLM GPU deployment, seconds.
     OCR_DEEPSEEK_TIMEOUT: float = 300.0
     # Native backend only: resolution at which pages without a text layer are
     # rendered before OCR. 200 suits tesseract; clamped to 72-600.

application/parser/file/anydoc_parser.py

@@ -68,6 +68,12 @@ ANYDOC_SUFFIXES: Tuple[str, ...] = (
 # else in ``ANYDOC_SUFFIXES`` is anydoc-only ("gained") and is mapped to a
 # fallback-less ``AnydocParser`` in every parser map.
 _CORE_SUFFIXES = frozenset({".pdf", ".docx", ".pptx", ".xlsx", ".csv"})
+
+# Spreadsheets anydoc may refuse on its fixed, non-configurable limits
+# (2M XML nodes per part, 128 MiB decompressed per part — a ~265k-row sheet
+# trips them) that the plain tabular parsers read fine in a few hundred MB.
+# For these a ResourceLimitError is a reason to delegate, not to fail.
+_TABULAR_SUFFIXES = frozenset({".csv", ".xlsx", ".xlsm", ".xlsb", ".xls", ".ods"})
 ANYDOC_GAINED_SUFFIXES: Tuple[str, ...] = tuple(
     suffix for suffix in ANYDOC_SUFFIXES if suffix not in _CORE_SUFFIXES
 )
@@ -171,9 +177,14 @@ class AnydocParser(BaseParser):
             content = anydoc.to_markdown(str(path))
         except anydoc.ResourceLimitError as exc:
             # anydoc refused because the file is too expensive (declared
-            # decompression size, nesting depth, node count). A heavier
-            # engine would spend exactly what was just refused, so this is
-            # terminal rather than a reason to delegate.
+            # decompression size, nesting depth, node count). For office
+            # documents a heavier engine would spend exactly what was just
+            # refused, so this is terminal. Spreadsheets are the exception:
+            # anydoc's cell/node limits sit below what ordinary business
+            # exports reach, and the tabular fallback (docling's gate hands
+            # oversized sheets to pandas/openpyxl) is lighter, not heavier.
+            if path.suffix.lower() in _TABULAR_SUFFIXES and self.fallback_parser is not None:
+                return self._delegate(path, errors, f"{type(exc).__name__}: {exc}")
             self.last_engine = None
             raise DocumentParseError(
                 f"Failed to parse {path.name}: {type(exc).__name__}: {exc}"
@@ -304,10 +315,13 @@ class AnydocParser(BaseParser):
         Runs only when the fallback parser both has OCR on and exposes
         ``ocr_pages`` (the native OCR PDF parser, or the docling PDF parser).
         With OCR off the fallback is the plain pypdf parser and mixed
-        documents keep today's behaviour: text pages only. An OCR failure is
-        logged and the anydoc text kept — a partial document beats a
-        rejected upload, and the loud-failure path stays reserved for fully
-        scanned files.
+        documents keep today's behaviour: text pages only. Pages are OCR'd
+        one at a time so a failure on one page costs that page, not the OCR
+        of every other; the anydoc text is always kept — a partial document
+        beats a rejected upload, and the loud-failure path stays reserved
+        for fully scanned files. anydoc's PDF markdown carries no page
+        separators, so the OCR'd pages are appended after the text pages
+        rather than spliced into position.
         """
         fallback = self.fallback_parser
         if not (getattr(fallback, "ocr_enabled", False) and hasattr(fallback, "ocr_pages")):
@@ -326,7 +340,6 @@ class AnydocParser(BaseParser):
         try:
             if not fallback.parser_config_set:
                 fallback.init_parser()
-            texts = fallback.ocr_pages(path, indices)
         except Exception:  # noqa: BLE001 - keep the text pages rather than fail the file
             logger.warning(
                 "OCR of the scanned pages of %s failed; indexing the text pages only",
@@ -334,7 +347,20 @@ class AnydocParser(BaseParser):
                 exc_info=True,
             )
             return content
-        extra = [texts[index] for index in indices if texts.get(index, "").strip()]
+        extra: List[str] = []
+        for index in indices:
+            try:
+                text = fallback.ocr_pages(path, [index]).get(index, "")
+            except Exception:  # noqa: BLE001 - one page's failure must not drop the others
+                logger.warning(
+                    "OCR of page %d of %s failed; skipping that page",
+                    index + 1,
+                    path.name,
+                    exc_info=True,
+                )
+                continue
+            if text and text.strip():
+                extra.append(text)
         if not extra:
             return content
         self._last_ocr_pages = (path, len(extra))

application/parser/file/docling_parser.py

@@ -543,6 +543,10 @@ class DoclingParser(BaseParser):
         vlm_options = vlm_model_specs.DEEPSEEKOCR_OLLAMA.model_copy(deep=True)
         vlm_options.url = settings.OCR_DEEPSEEK_URL
         vlm_options.params["model"] = settings.OCR_DEEPSEEK_MODEL
+        # docling's spec default is 90 s per request, which a laptop-hosted
+        # 3B model overruns; honour the same per-page budget the native
+        # backend gives the endpoint.
+        vlm_options.timeout = float(settings.OCR_DEEPSEEK_TIMEOUT)
         pipeline_options = VlmPipelineOptions(
             vlm_options=vlm_options, enable_remote_services=True
         )
@@ -622,7 +626,12 @@ class DoclingParser(BaseParser):
         tesseract's CJK models space every glyph ("互相 保密 协议"); the native
         backend collapses that in ``TesseractEngine`` and the docling path
         needs the same treatment or Chinese scans index worse here than there.
-        Other engines (and OCR off) return the export untouched.
+        Other engines (and OCR off) return the export untouched. Only the PDF
+        and image parsers construct with OCR on — the office/markup parsers
+        never OCR, so their born-digital text must not be rewritten. On a
+        hybrid-OCR PDF the collapse still covers the whole export, native
+        text included; deliberate spaces inside CJK runs are rare enough
+        there that indexing scans correctly wins.
         """
         if self.ocr_enabled and self._active_ocr_engine == "tesseract":
             return collapse_cjk_spaces(content)
@@ -866,21 +875,21 @@ class DoclingDocxParser(DoclingParser):
     """Docling-based DOCX parser."""
 
     def __init__(self):
-        super().__init__(export_format="markdown")
+        super().__init__(ocr_enabled=False, export_format="markdown")
 
 
 class DoclingPPTXParser(DoclingParser):
     """Docling-based PPTX parser."""
 
     def __init__(self):
-        super().__init__(export_format="markdown")
+        super().__init__(ocr_enabled=False, export_format="markdown")
 
 
 class DoclingXLSXParser(DoclingParser):
     """Docling-based XLSX parser with table structure."""
 
     def __init__(self):
-        super().__init__(table_structure=True, export_format="markdown")
+        super().__init__(ocr_enabled=False, table_structure=True, export_format="markdown")
 
     def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
         """Parse an XLSX file, delegating oversized files to ``ExcelParser``."""
@@ -899,7 +908,7 @@ class DoclingHTMLParser(DoclingParser):
     """Docling-based HTML parser."""
 
     def __init__(self):
-        super().__init__(export_format="markdown")
+        super().__init__(ocr_enabled=False, export_format="markdown")
 
     def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
         """Parse HTML, truncating oversized element-dense markup first."""
@@ -933,7 +942,7 @@ class DoclingCSVParser(DoclingParser):
     """Docling-based CSV parser."""
 
     def __init__(self):
-        super().__init__(table_structure=True, export_format="markdown")
+        super().__init__(ocr_enabled=False, table_structure=True, export_format="markdown")
 
     def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
         """Parse a CSV file, delegating oversized files to the plain ``CSVParser``."""
@@ -952,21 +961,21 @@ class DoclingMarkdownParser(DoclingParser):
     """Docling-based Markdown parser."""
 
     def __init__(self):
-        super().__init__(export_format="markdown")
+        super().__init__(ocr_enabled=False, export_format="markdown")
 
 
 class DoclingAsciiDocParser(DoclingParser):
     """Docling-based AsciiDoc parser."""
 
     def __init__(self):
-        super().__init__(export_format="markdown")
+        super().__init__(ocr_enabled=False, export_format="markdown")
 
 
 class DoclingVTTParser(DoclingParser):
     """Docling-based WebVTT (video text tracks) parser."""
 
     def __init__(self):
-        super().__init__(export_format="markdown")
+        super().__init__(ocr_enabled=False, export_format="markdown")
 
     def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
         """Parse WebVTT, truncating oversized cue-dense tracks first."""
@@ -977,4 +986,4 @@ class DoclingXMLParser(DoclingParser):
     """Docling-based XML parser (USPTO, JATS)."""
 
     def __init__(self):
-        super().__init__(export_format="markdown")
+        super().__init__(ocr_enabled=False, export_format="markdown")

application/parser/file/ocr_parser.py

@@ -56,6 +56,12 @@ DEFAULT_TEXT_LAYER_MIN_CHARS = 32
 _DEFAULT_MIN_CHARS_PER_PAGE = 20
 _DEFAULT_RENDER_DPI = 200
 _MIN_RENDER_DPI, _MAX_RENDER_DPI = 72, 600
+# Pixel budget for one rendered page. pypdfium2 allocates width*height*4
+# bytes up front with no cap of its own, so a 14400x14400 pt page (the PDF
+# maximum; a 191-byte file) at 200 dpi would ask for 40000x40000 px = 6 GiB.
+# 40 MP is ~10x a Letter page at 200 dpi and still OCRs an A0 poster at a
+# usable ~125 dpi; anything larger renders at the scale that fits.
+_MAX_RENDER_PIXELS = 40_000_000
 _TESSERACT_TIMEOUT_SECONDS = 300
 # docling's DeepSeek-OCR prompt minus its ``<|grounding|>`` prefix: grounding
 # makes the model wrap every element in ref/det tags with bounding boxes,
@@ -368,9 +374,32 @@ def build_native_ocr_engine(engine: Optional[str] = None, languages: Optional[Li
 # ---------------------------------------------------------------------------
 
 
+def render_scale(page_width_pt: float, page_height_pt: float, dpi: int) -> float:
+    """Render scale for a page of the given size at ``dpi``, capped by ``_MAX_RENDER_PIXELS``.
+
+    The cap is what stands between an oversized (or hostile) MediaBox and a
+    multi-gigabyte bitmap allocation in the ingest worker.
+    """
+    scale = dpi / 72.0
+    area = max(1.0, float(page_width_pt)) * max(1.0, float(page_height_pt))
+    max_scale = (_MAX_RENDER_PIXELS / area) ** 0.5
+    return min(scale, max_scale)
+
+
 def _render_page(page, dpi: int):
-    """Render a pypdfium2 page to a PIL image at ``dpi``."""
-    return page.render(scale=dpi / 72.0).to_pil()
+    """Render a pypdfium2 page to a PIL image at ``dpi``, within the pixel budget."""
+    width_pt, height_pt = page.get_size()
+    scale = render_scale(width_pt, height_pt, dpi)
+    if scale < dpi / 72.0:
+        logger.warning(
+            "Page is %.0fx%.0f pt; rendering at %.0f dpi instead of %d to stay within %d MP",
+            width_pt,
+            height_pt,
+            scale * 72.0,
+            dpi,
+            _MAX_RENDER_PIXELS // 1_000_000,
+        )
+    return page.render(scale=scale).to_pil()
 
 
 def text_layer_counts(path: Path) -> List[int]:
@@ -399,20 +428,54 @@ def text_layer_counts(path: Path) -> List[int]:
     return counts
 
 
+def _page_has_image(page) -> bool:
+    """Whether the page draws at least one image XObject (something OCR could read)."""
+    import pypdfium2.raw as pdfium_c
+
+    try:
+        return next(iter(page.get_objects(filter=(pdfium_c.FPDF_PAGEOBJ_IMAGE,))), None) is not None
+    except Exception:  # noqa: BLE001 - treat an unreadable page as image-bearing
+        return True
+
+
 def scanned_page_indices(path: Path, min_chars: int = DEFAULT_TEXT_LAYER_MIN_CHARS) -> List[int]:
-    """0-based indices of the pages whose text layer is too thin to be real.
+    """0-based indices of the pages that look scanned: no text layer, or a thin one over an image.
 
     The probe behind mixed-document handling: a text-layer converter (anydoc)
     reads the text pages and silently skips these, so whoever owns OCR has to
-    fill them in. Returns [] when the file cannot be probed — the caller then
-    keeps what it has rather than failing a successful parse.
+    fill them in. A page with no text at all always qualifies. A page with a
+    thin text layer (under ``min_chars``: a Bates stamp on a scan, a page
+    number under a full-page figure) qualifies only when it also draws an
+    image — a cover page reading just "Annual Report 2025" has nothing to OCR,
+    and OCR-ing it would duplicate text the converter already read. Returns []
+    when the file cannot be probed — the caller then keeps what it has rather
+    than failing a successful parse.
     """
+    import pypdfium2 as pdfium
+
+    floor = max(1, int(min_chars))
+    indices: List[int] = []
     try:
-        counts = text_layer_counts(Path(path))
+        pdf = pdfium.PdfDocument(str(path))
+        try:
+            for index in range(len(pdf)):
+                page = pdf[index]
+                try:
+                    textpage = page.get_textpage()
+                    try:
+                        count = max(0, textpage.count_chars())
+                    finally:
+                        textpage.close()
+                    if count == 0 or (count < floor and _page_has_image(page)):
+                        indices.append(index)
+                finally:
+                    page.close()
+        finally:
+            pdf.close()
     except Exception:  # noqa: BLE001 - a probe must never fail the parse
         logger.warning("Could not probe %s for scanned pages", Path(path).name, exc_info=True)
         return []
-    return [index for index, count in enumerate(counts) if count < max(1, int(min_chars))]
+    return indices
 
 
 def _check_near_empty(name: str, engine: str, content: str, pages: int, ocr_pages: int) -> None:

application/parser/file/pdf_trust.py

@@ -27,10 +27,9 @@ silent-drop cases with zero false positives on the other 14 PDFs, and cost
 import re
 import zlib
 from pathlib import Path
-from typing import Dict, Iterator, List, Union
+from typing import Dict, Iterator, List, Optional, Union
 
 _OBJ = re.compile(rb"\d+\s+\d+\s+obj(.*?)endobj", re.DOTALL)
-_STREAM = re.compile(rb"stream\r?\n(.*?)\r?\nendstream", re.DOTALL)
 _TYPE0 = re.compile(rb"/Subtype\s*/Type0")
 # The CJK expectation is keyed on CIDSystemInfo /Ordering ALONE — the
 # authoritative "this PDF maps text through a CJK character collection"
@@ -49,6 +48,7 @@ _WINDOW_BEFORE, _WINDOW_AFTER = 200, 800
 # Extracted text with fewer CJK characters than this, from a PDF that
 # declares CJK fonts, is treated as a silent drop.
 _MIN_CJK_CHARS = 10
+_CJK_PROBLEM_PREFIX = "PDF declares CJK fonts"
 
 _CJK_RANGES = (
     ("一", "鿿"),  # CJK Unified Ideographs
@@ -64,13 +64,46 @@ _CJK_RANGES = (
 _STREAM_INFLATE_CAP = 4_000_000
 
 
+def _stream_bodies(raw: bytes) -> Iterator[bytes]:
+    """The raw bytes between each ``stream`` / ``endstream`` keyword pair.
+
+    A ``find`` walk rather than a regex: ``stream\\r?\\n(.*?)\\r?\\nendstream``
+    misses every stream whose data runs straight into ``endstream`` with no
+    EOL (common in the wild), and when it misses one its lazy group scans on
+    to the *next* stream's terminator — quadratic on large files (a 10 MB
+    file measured 32 s, matching a fifth of its streams) and a merged,
+    undecodable body for the rest.
+    """
+    pos = 0
+    while True:
+        pos = raw.find(b"stream", pos)
+        if pos < 0:
+            return
+        if raw[max(0, pos - 3):pos] == b"end":
+            pos += 6
+            continue
+        start = pos + 6
+        if raw[start:start + 2] == b"\r\n":
+            start += 2
+        elif raw[start:start + 1] in (b"\n", b"\r"):
+            start += 1
+        end = raw.find(b"endstream", start)
+        if end < 0:
+            return
+        body = raw[start:end]
+        if body.endswith(b"\r\n"):
+            body = body[:-2]
+        elif body.endswith((b"\n", b"\r")):
+            body = body[:-1]
+        pos = end + 9
+        yield body
+
+
 def _decompressed_streams(raw: bytes) -> Iterator[bytes]:
     """Each Flate stream in ``raw`` that decompresses, one at a time, capped."""
-    for match in _STREAM.finditer(raw):
+    for body in _stream_bodies(raw):
         try:
-            inflated = zlib.decompressobj().decompress(
-                match.group(1), _STREAM_INFLATE_CAP
-            )
+            inflated = zlib.decompressobj().decompress(body, _STREAM_INFLATE_CAP)
         except zlib.error:
             continue
         yield inflated
@@ -162,16 +195,65 @@ def verify_extraction(data: bytes, markdown: str) -> List[str]:
         cjk = _cjk_chars(markdown, _MIN_CJK_CHARS)
         if cjk < _MIN_CJK_CHARS:
             problems.append(
-                f"PDF declares CJK fonts but the extracted text contains only "
+                f"{_CJK_PROBLEM_PREFIX} but the extracted text contains only "
                 f"{cjk} CJK character(s)"
             )
     return problems
 
 
+def _text_layer_cjk_chars(file: Path, up_to: int) -> Optional[int]:
+    """CJK characters in the text layer as pypdfium2 reads it, stopping at ``up_to``.
+
+    pdfium ships Adobe's predefined CMaps, so it is an independent reference
+    for whether the document *has* CJK text to lose. None when pypdfium2 is
+    unavailable or the file cannot be read that way.
+    """
+    try:
+        import pypdfium2 as pdfium
+    except ImportError:
+        return None
+    try:
+        pdf = pdfium.PdfDocument(str(file))
+    except Exception:  # noqa: BLE001 - a reference probe, not a parse
+        return None
+    count = 0
+    try:
+        for index in range(len(pdf)):
+            page = pdf[index]
+            try:
+                textpage = page.get_textpage()
+                try:
+                    text = textpage.get_text_bounded()
+                finally:
+                    textpage.close()
+            finally:
+                page.close()
+            count += _cjk_chars(text, up_to - count)
+            if count >= up_to:
+                break
+    except Exception:  # noqa: BLE001
+        return None
+    finally:
+        pdf.close()
+    return count
+
+
 def verify_pdf_file(file: Path, markdown: str) -> List[str]:
-    """``verify_extraction`` for a file on disk; unreadable files trust the output."""
+    """``verify_extraction`` for a file on disk; unreadable files trust the output.
+
+    The CJK cross-check is confirmed against pdfium's own text layer: a
+    stray ``/Ordering (Japan1)`` in an otherwise Latin document (a 48-page
+    Quartz export in the corpus carries one among 258 Identity fonts) is not
+    a dropped Chinese column, and must not send the file to a full docling
+    re-parse.
+    """
     try:
         data = Path(file).read_bytes()
     except OSError:
         return []
-    return verify_extraction(data, markdown)
+    problems = verify_extraction(data, markdown)
+    if any(problem.startswith(_CJK_PROBLEM_PREFIX) for problem in problems):
+        reference = _text_layer_cjk_chars(Path(file), _MIN_CJK_CHARS)
+        if reference is not None and reference < _MIN_CJK_CHARS:
+            problems = [p for p in problems if not p.startswith(_CJK_PROBLEM_PREFIX)]
+    return problems