ocr update

tests/parser/file/fixtures/nda_en_zh_cid_font.pdf

@@ -0,0 +1,101 @@
+%PDF-1.4
+%“Œ‹ž ReportLab Generated PDF document (opensource)
+1 0 obj
+<<
+/F1 2 0 R /F2 3 0 R /F3 4 0 R /F4 5 0 R
+>>
+endobj
+2 0 obj
+<<
+/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
+>>
+endobj
+3 0 obj
+<<
+/BaseFont /Times-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
+>>
+endobj
+4 0 obj
+<<
+/BaseFont /STSong-Light /DescendantFonts [ <<
+/BaseFont /STSong-Light /CIDSystemInfo <<
+/Ordering (GB1) /Registry (Adobe) /Supplement 0
+>> /DW 1000 /FontDescriptor <<
+/Ascent 752 /CapHeight 737 /Descent -271 /Flags 6 /FontBBox [ -25 -254 1000 880 ] /FontName /STSongStd-Light 
+  /ItalicAngle 0 /Leading 148 /MaxWidth 1000 /MissingWidth 500 /StemH 91 /StemV 58 
+  /Type /FontDescriptor /XHeight 553
+>> /Subtype /CIDFontType0 /Type /Font 
+  /W [ 1 [ 207 270 342 467 462 797 710 239 374 ] 10 [ 374 423 605 238 375 238 334 462 ] 18 26 462 27 28 238 
+  29 31 605 32 [ 344 748 684 560 695 739 563 511 729 793 
+  318 312 666 526 896 758 772 544 772 628 
+  465 607 753 711 972 647 620 607 374 333 
+  374 606 500 239 417 503 427 529 415 264 
+  444 518 241 230 495 228 793 527 524 ] 81 [ 524 504 338 336 277 517 450 652 466 452 
+  407 370 258 370 605 ] ]
+>> ] /Encoding /UniGB-UCS2-H /Name /F3 /Subtype /Type0 /Type /Font
+>>
+endobj
+5 0 obj
+<<
+/BaseFont /Times-Roman /Encoding /WinAnsiEncoding /Name /F4 /Subtype /Type1 /Type /Font
+>>
+endobj
+6 0 obj
+<<
+/Contents 10 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 9 0 R /Resources <<
+/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
+>> /Rotate 0 /Trans <<
+
+>> 
+  /Type /Page
+>>
+endobj
+7 0 obj
+<<
+/PageMode /UseNone /Pages 9 0 R /Type /Catalog
+>>
+endobj
+8 0 obj
+<<
+/Author (\(anonymous\)) /CreationDate (D:20260822115950+02'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260822115950+02'00') /Producer (ReportLab PDF Library - \(opensource\)) 
+  /Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
+>>
+endobj
+9 0 obj
+<<
+/Count 1 /Kids [ 6 0 R ] /Type /Pages
+>>
+endobj
+10 0 obj
+<<
+/Filter [ /ASCII85Decode /FlateDecode ] /Length 1647
+>>
+stream
+Gatm<gN):C&:O:SoKt6S$HmLQ&3Rqa-FV*"%Nm-aMW-4=#+,j_=@nmi2Z)t-)*e6=$[Ld,&=Q79:X>PPVJQYC5?'k&+i&d>B>799%,;FkdkJW*5heHuF@91dXh%AJZ6fb3i'sKhCnMdY'0VkEO4GA.p;"DJY62!"YAIse&.WT60%ek]="]O%+5Ot<#Fl%^o]il7fm2dGp=dm)m^:gDD9"d?pER14WDHmu']GE^2k$Vs`2KagWk")2f26\M]Bs+K@uNQtga3OQkLQ7H-hY4>1!4.4XoK+nl.ZlK#$d"IN+=9q.'8T'8cF3g*[-3:LS5c>fi!l0mSVa4QpjYgA<$0LC`2-c:@^;pCa)ekN>!I%&S3he7oPPIfI^JJ8"'-"K,>QL6DaesnS'ThOk/)63Q<B9b;^E^!3N(^L"g8T-_[_e%ED-JKcd/L:T#=nOk`'\l#7Q=,(RK;`0#S<T4BU:`HhPGE"*QrTjD+LciNhe\[arIY[:*6r!Z(b46[T;c]'eT2ib4:#PYiS4KbfJkaWh^D3%L,2kV2tNtL-eVVV!V=WcZC*LmZ1JF,Q`_WpI/i"9jUr1Gd(ISB[Id?[]g@-RLQD"d?"iI[81]"&B'nLTX>D<J'G5>]1d]+<&nMsjUuVm%-`_Dn^N7Y3@(%aO`>?OaQR7"S`I)*4n^gDS'@E((C?3'qDA#_!0S=@#*`D<h%J/PD"f$4CY;-(VXO;.Ngc^I+,-r\]qpUQHC*nl`bn!js*R3SkJqAMhH<a9;!@-5%1$9AH:,n;b&'!\_DddM7mc;^=s`,BOb&%SnDA7Rl;HLN*t%(`jGJS&".8T-M[a.`>blJkd8M[QOjG@Lffk@WSsIihjLDN`q?@n!YaFRc6etUSZ3TeZg#m?mL-MTMO5V)b]\Em&*E2^jCc`Xsk1<"JLgnj,EWu^Kg\"3.u<u5YCaES/ZDf5DpeMd24qDipat:7Oab*_jqT_s6MJ(i_J60;p<ik,/MGX2+u90)Xt!desS2A+)qeT6sUK/IJ'C=1<0u0>SaqZ<FhNoMfaf*!B2`NiLP;eB4?[`;/Y%o&Gm)'qO1JK0uckb!lh3>=t73#j/BK3<dLaZd96%gq]>W+8,V.,0D<(=M`#OG+,1-Fh</5[IM)#q0>k-!*=d[eI7&!b)UOUK$3XO.Y5HWgM[D<SAa`SCR9MN]H2Q[$5(+ZDUq;e-9V7T89Qo[Rm`4fG0o4@V.p48_A)/:trd:.`nh*S%-%oo[?Go6OFLjPL]dfcTW*/@I<%u%sqi96p7;=;:bYI+:AU5':P>$c<"Wh_I_.r/0:j+[",VmP^3"R7]Vuj1Bju_M4fQpgg+K1-[?e$i#W^p3[3m^DdEM$S$d\G^r%4.<YC,K9<kt81fnor-c7+%aA'BW<5M_bT$PN_@IR!Yssf=Ie",juu4&#3f]0+Uq".M]_sN3S\fN4OR!lJL2(a=:b4Ff&7E;q)O_P#WqT?;^@'e:m]2q4Y\']#rYS`0J,.E<b/Dai'5?mlc6AhJna\JGg_:Yc:98:nsRLal!#c+EsM(<tb(J^cTEVjq!Ll`V)/=\n1?2VP!nUibVne=IuE;BNdBFoimp4D_?%=$%6*-QW3f0rnX&<H85'1fg=@QlClK`^N0$X\\#1DFp#6kjo8*>*l66RfIdu>~>endstream
+endobj
+xref
+0 11
+0000000000 65535 f 
+0000000061 00000 n 
+0000000122 00000 n 
+0000000229 00000 n 
+0000000337 00000 n 
+0000001270 00000 n 
+0000001379 00000 n 
+0000001583 00000 n 
+0000001651 00000 n 
+0000001931 00000 n 
+0000001990 00000 n 
+trailer
+<<
+/ID 
+[<6167f87b12d550221d4edd2d2c131d94><6167f87b12d550221d4edd2d2c131d94>]
+% ReportLab generated PDF document -- digest (opensource)
+
+/Info 8 0 R
+/Root 7 0 R
+/Size 11
+>>
+startxref
+3729
+%%EOF

tests/parser/file/test_anydoc_parser.py

@@ -27,10 +27,14 @@ from application.parser.file.anydoc_parser import (  # noqa: E402 — after impo
 )
 
 
+# Long enough to clear the scanned-PDF near-empty guard (_MIN_SCAN_FALLBACK_CHARS).
+FALLBACK_TEXT = "fallback parser text output, long enough to clear the scanned-PDF guard."
+
+
 class _RecordingFallback(BaseParser):
     """Stands in for docling / a legacy parser so delegation is observable."""
 
-    def __init__(self, result="fallback text"):
+    def __init__(self, result=FALLBACK_TEXT):
         super().__init__(parser_config={})
         self.calls = []
         self._result = result
@@ -146,7 +150,7 @@ def test_scanned_pdf_delegates_to_fallback(tmp_path):
 
     out = parser.parse_file(path)
 
-    assert out == "fallback text"
+    assert out == FALLBACK_TEXT
     assert fallback.calls == [path]
     assert parser.last_engine == "_RecordingFallback"
 
@@ -229,7 +233,7 @@ def test_typed_convert_error_delegates(tmp_path, monkeypatch):
     path = tmp_path / "x.pdf"
     path.write_bytes(b"%PDF-1.4")
 
-    assert AnydocParser(fallback_parser=fallback).parse_file(path) == "fallback text"
+    assert AnydocParser(fallback_parser=fallback).parse_file(path) == FALLBACK_TEXT
 
 
 def test_typed_convert_error_message_reaches_the_user(tmp_path, monkeypatch):
@@ -311,3 +315,222 @@ def test_init_parser_imports_for_real_not_just_find_spec(monkeypatch):
 
     with pytest.raises(ImportError, match="firecrawl-anydoc"):
         AnydocParser().init_parser()
+
+
+# --- PDF trust check + tableize wiring (PR 3) -----------------------------------
+
+from pathlib import Path as _Path
+
+FIXTURES = _Path(__file__).parent / "fixtures"
+CID_PDF = FIXTURES / "nda_en_zh_cid_font.pdf"
+
+
+# Long enough to clear the near-empty guard a trust-check re-parse must pass.
+_REROUTE_TEXT = "docling reroute text, long enough to clear the near-empty reroute guard"
+
+
+class _FakeDoclingFallback:
+    """Registered as a DoclingParser subclass so ``_is_docling_backed`` is True."""
+
+    def __new__(cls):
+        from application.parser.file.docling_parser import DoclingParser
+
+        class _Inner(DoclingParser):
+            def __init__(self):
+                self._parser_config = {}
+                self.calls = []
+                self.last_engine = None
+
+            def parse_file(self, file, errors="ignore"):
+                self.calls.append(file)
+                return _REROUTE_TEXT
+
+        return _Inner()
+
+
+def test_trust_flagged_pdf_reroutes_to_docling_fallback():
+    fallback = _FakeDoclingFallback()
+    parser = AnydocParser(fallback_parser=fallback)
+
+    out = parser.parse_file(CID_PDF)
+
+    assert out == _REROUTE_TEXT
+    assert fallback.calls == [CID_PDF]
+    assert parser.last_engine == "_Inner"
+    assert parser.get_file_metadata(CID_PDF) == {}  # rerouted, nothing to warn about
+
+
+def test_trust_flagged_pdf_without_docling_keeps_output_and_warns():
+    parser = AnydocParser(fallback_parser=_RecordingFallback())  # not docling-backed
+
+    out = parser.parse_file(CID_PDF)
+
+    assert "NON-DISCLOSURE" in out.upper() or len(out) > 50  # anydoc's own output kept
+    meta = parser.get_file_metadata(CID_PDF)
+    assert "parse_warnings" in meta
+    assert any("ToUnicode" in w for w in meta["parse_warnings"])
+    assert any("CJK" in w for w in meta["parse_warnings"])
+    assert parser.last_engine == "anydoc"
+
+
+def test_trust_check_disabled_stamps_nothing(monkeypatch):
+    from application.parser.file import anydoc_parser as ap
+
+    monkeypatch.setattr(ap.settings, "PDF_TRUST_CHECK", False)
+    parser = AnydocParser()
+
+    parser.parse_file(CID_PDF)
+
+    assert parser.get_file_metadata(CID_PDF) == {}
+
+
+def test_trust_reroute_failure_keeps_anydoc_output():
+    fallback = _FakeDoclingFallback()
+
+    def _boom(file, errors="ignore"):
+        raise RuntimeError("docling exploded")
+
+    fallback.parse_file = _boom
+    parser = AnydocParser(fallback_parser=fallback)
+
+    out = parser.parse_file(CID_PDF)
+
+    assert "docling" not in out
+    assert "parse_warnings" in parser.get_file_metadata(CID_PDF)
+    assert parser.last_engine == "anydoc"
+
+
+def test_trust_reroute_near_empty_keeps_anydoc_output():
+    """A docling pipeline dropout ('' / '<!-- image -->') returns without
+    raising; adopting it would swap anydoc's real text for an empty document."""
+    fallback = _FakeDoclingFallback()
+    fallback.parse_file = lambda file, errors="ignore": "<!-- image -->"
+    parser = AnydocParser(fallback_parser=fallback)
+
+    out = parser.parse_file(CID_PDF)
+
+    assert "<!-- image -->" not in out
+    assert "parse_warnings" in parser.get_file_metadata(CID_PDF)
+    assert parser.last_engine == "anydoc"
+
+
+def test_warnings_reset_between_files(tmp_path):
+    parser = AnydocParser()
+    parser.parse_file(CID_PDF)
+    assert parser.get_file_metadata(CID_PDF) != {}
+
+    clean = tmp_path / "clean.csv"
+    clean.write_text("a,b\n1,2\n")
+    parser.parse_file(clean)
+
+    assert parser.get_file_metadata(clean) == {}
+    assert parser.get_file_metadata(CID_PDF) == {}
+
+
+def test_trust_check_errors_never_fail_the_parse(monkeypatch, tmp_path):
+    def _explode(path, markdown):
+        raise RuntimeError("scanner bug")
+
+    import application.parser.file.pdf_trust as pt
+
+    monkeypatch.setattr(pt, "verify_pdf_file", _explode)
+    _fake_anydoc(monkeypatch, lambda path: "# converted fine")
+    path = tmp_path / "x.pdf"
+    path.write_bytes(b"%PDF-1.4 irrelevant")
+
+    assert AnydocParser().parse_file(path) == "# converted fine"
+
+
+def test_tableize_applied_when_enabled(monkeypatch, tmp_path):
+    from application.parser.file import anydoc_parser as ap
+
+    monkeypatch.setattr(ap.settings, "ANYDOC_TABLEIZE", True)
+    monkeypatch.setattr(ap.settings, "PDF_TRUST_CHECK", False)
+    _fake_anydoc(
+        monkeypatch,
+        lambda path: "Cash ..... 1,234 900\nDebt ..... 2,000 1,500\nEquity ..... 900 800",
+    )
+    path = tmp_path / "x.pdf"
+    path.write_bytes(b"%PDF-1.4 irrelevant")
+
+    out = AnydocParser().parse_file(path)
+
+    assert "| Cash | 1,234 | 900 |" in out
+
+
+def test_tableize_disabled_by_default(monkeypatch, tmp_path):
+    from application.parser.file import anydoc_parser as ap
+
+    monkeypatch.setattr(ap.settings, "PDF_TRUST_CHECK", False)
+    flat = "Cash ..... 1,234 900\nDebt ..... 2,000 1,500\nEquity ..... 900 800"
+    _fake_anydoc(monkeypatch, lambda path: flat)
+    path = tmp_path / "x.pdf"
+    path.write_bytes(b"%PDF-1.4 irrelevant")
+
+    assert AnydocParser().parse_file(path) == flat
+
+
+def test_tableize_never_touches_docling_reroute(monkeypatch):
+    from application.parser.file import anydoc_parser as ap
+
+    monkeypatch.setattr(ap.settings, "ANYDOC_TABLEIZE", True)
+    fallback = _FakeDoclingFallback()
+    parser = AnydocParser(fallback_parser=fallback)
+
+    out = parser.parse_file(CID_PDF)
+
+    assert out == _REROUTE_TEXT  # verbatim, not post-processed
+
+
+# --- scanned-PDF near-empty guard (the OCR_ENGINE seam's loud-failure side) ------
+
+
+def test_scanned_pdf_with_near_empty_fallback_fails_loudly_when_ocr_off(tmp_path):
+    """A scan whose fallback (OCR off) extracts almost nothing must fail with
+    an actionable message, not be stored as an empty document."""
+    path = _scanned_pdf(tmp_path / "scan.pdf")
+    fallback = _RecordingFallback(result="   ")
+    fallback.ocr_enabled = False
+    parser = AnydocParser(fallback_parser=fallback)
+
+    with pytest.raises(DocumentParseError, match="DOCLING_OCR_ENABLED"):
+        parser.parse_file(path)
+    assert parser.last_engine is None
+
+
+def test_scanned_pdf_with_near_empty_fallback_and_ocr_on_reports_it(tmp_path):
+    path = _scanned_pdf(tmp_path / "scan.pdf")
+    fallback = _RecordingFallback(result="x")
+    fallback.ocr_enabled = True
+
+    with pytest.raises(DocumentParseError, match="even with OCR enabled"):
+        AnydocParser(fallback_parser=fallback).parse_file(path)
+
+
+def test_scanned_pdf_with_substantial_fallback_output_passes(tmp_path):
+    """docling extracting a text layer anydoc refused (the HK-bill case) must
+    keep working — the guard only fires on near-empty results."""
+    path = _scanned_pdf(tmp_path / "scan.pdf")
+    out = AnydocParser(fallback_parser=_RecordingFallback()).parse_file(path)
+    assert out == FALLBACK_TEXT
+
+
+def test_non_scan_refusals_do_not_trigger_the_guard(tmp_path, monkeypatch):
+    """Only the OCR-required refusal implies 'content exists but needs OCR';
+    a malformed file with a short fallback result stays a successful parse."""
+    fake = _fake_anydoc(monkeypatch, None)
+
+    class MalformedError(fake.ConvertError):
+        pass
+
+    fake.MalformedError = MalformedError
+
+    def _refuse(path):
+        raise MalformedError("structurally unusable")
+
+    fake.to_markdown = _refuse
+    path = tmp_path / "x.docx"
+    path.write_bytes(b"irrelevant")
+
+    out = AnydocParser(fallback_parser=_RecordingFallback(result="tiny")).parse_file(path)
+    assert out == "tiny"

tests/parser/file/test_bulk.py

@@ -628,3 +628,115 @@ class TestParserEngineSwitch:
         assert ".xhtml" in extractor
         for suffix in (".pdf", ".docx", ".csv", ".xlsx", ".html", ".xhtml", ".pptx"):
             extractor[suffix].init_parser()
+
+
+# =====================================================================
+# Gained formats (anydoc-only: legacy/macro Office, OpenDocument, RTF)
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestGainedFormats:
+    def test_every_supported_extension_has_a_parser_under_both_engines(self):
+        """The invariant that keeps constants.py and the parser maps in lockstep.
+
+        Without it, an extension accepted at upload but missing from the map
+        falls to SimpleDirectoryReader's plain-text read — for a binary
+        format that means mojibake silently ingested and embedded.
+        (`.txt` is the one deliberate plain-text read.)
+        """
+        pytest.importorskip("anydoc")
+        from application.parser.file.bulk import get_default_file_extractor
+        from application.parser.file.constants import (
+            SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS,
+        )
+
+        for engine in ("anydoc", "docling"):
+            extractor = get_default_file_extractor(engine=engine)
+            missing = [
+                suffix
+                for suffix in SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS
+                if suffix != ".txt" and suffix not in extractor
+            ]
+            assert missing == [], (engine, missing)
+
+    def test_gained_formats_map_to_anydoc_under_both_engines(self):
+        pytest.importorskip("anydoc")
+        from application.parser.file.anydoc_parser import (
+            ANYDOC_GAINED_SUFFIXES,
+            AnydocParser,
+        )
+        from application.parser.file.bulk import get_default_file_extractor
+
+        for engine in ("anydoc", "docling"):
+            extractor = get_default_file_extractor(engine=engine)
+            for suffix in ANYDOC_GAINED_SUFFIXES:
+                assert isinstance(extractor[suffix], AnydocParser), (engine, suffix)
+
+    def test_gained_formats_never_fall_back_to_anydoc_itself(self):
+        pytest.importorskip("anydoc")
+        from application.parser.file.anydoc_parser import AnydocParser
+        from application.parser.file.bulk import get_default_file_extractor
+
+        extractor = get_default_file_extractor(engine="anydoc")
+        assert extractor[".doc"].fallback_parser is None
+        assert extractor[".rtf"].fallback_parser is None
+        # ...while the core five keep their real fallback.
+        assert extractor[".pdf"].fallback_parser is not None
+        assert not isinstance(extractor[".pdf"].fallback_parser, AnydocParser)
+
+    def test_gained_entries_absent_without_anydoc(self, monkeypatch):
+        import sys
+
+        from application.parser.file.bulk import get_default_file_extractor
+
+        monkeypatch.setitem(sys.modules, "anydoc", None)
+        extractor = get_default_file_extractor(engine="docling")
+        assert ".doc" not in extractor  # degrades to the pre-anydoc map
+
+    def test_rtf_converts_end_to_end(self, tmp_path):
+        pytest.importorskip("anydoc")
+        from application.parser.file.bulk import get_default_file_extractor
+
+        path = tmp_path / "note.rtf"
+        path.write_text(r"{\rtf1\ansi Hello {\b bold} world.\par Second paragraph.}")
+        parser = get_default_file_extractor()[".rtf"]
+        parser.init_parser()
+
+        out = parser.parse_file(path)
+
+        assert "Hello **bold** world." in out
+        assert "Second paragraph." in out
+
+    def test_odt_converts_end_to_end(self, tmp_path):
+        pytest.importorskip("anydoc")
+        import zipfile
+
+        from application.parser.file.bulk import get_default_file_extractor
+
+        path = tmp_path / "doc.odt"
+        with zipfile.ZipFile(path, "w") as z:
+            z.writestr(
+                "mimetype",
+                "application/vnd.oasis.opendocument.text",
+                compress_type=zipfile.ZIP_STORED,
+            )
+            z.writestr(
+                "META-INF/manifest.xml",
+                '<?xml version="1.0"?><manifest:manifest xmlns:manifest="urn:oasis:names:tc:opendocument:xmlns:manifest:1.0">'
+                '<manifest:file-entry manifest:full-path="/" manifest:media-type="application/vnd.oasis.opendocument.text"/>'
+                '<manifest:file-entry manifest:full-path="content.xml" manifest:media-type="text/xml"/></manifest:manifest>',
+            )
+            z.writestr(
+                "content.xml",
+                '<?xml version="1.0"?><office:document-content xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" '
+                'xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0"><office:body><office:text>'
+                '<text:h text:outline-level="1">Title</text:h><text:p>Body text here.</text:p>'
+                "</office:text></office:body></office:document-content>",
+            )
+        parser = get_default_file_extractor()[".odt"]
+
+        out = parser.parse_file(path)
+
+        assert "# Title" in out
+        assert "Body text here." in out

tests/parser/file/test_docling_parser.py

@@ -1,6 +1,6 @@
 """Comprehensive tests for application/parser/file/docling_parser.py
 
-Covers: DoclingParser (init, _init_parser, _get_ocr_options, _export_content,
+Covers: DoclingParser (init, _init_parser, OCR engine selection, _export_content,
 parse_file), subclass initialization, error handling.
 """
 
@@ -27,8 +27,8 @@ class TestDoclingParserInit:
         assert parser.ocr_enabled is True
         assert parser.table_structure is True
         assert parser.export_format == "markdown"
-        assert parser.use_rapidocr is True
-        assert parser.ocr_languages == ["english"]
+        assert parser.ocr_engine is None  # None -> settings.OCR_ENGINE at build
+        assert parser.ocr_languages is None  # None -> the engine's own default
         assert parser.force_full_page_ocr is False
         assert parser._converter is None
 
@@ -39,14 +39,14 @@ class TestDoclingParserInit:
             ocr_enabled=False,
             table_structure=False,
             export_format="text",
-            use_rapidocr=False,
+            ocr_engine="rapidocr",
             ocr_languages=["german"],
             force_full_page_ocr=True,
         )
         assert parser.ocr_enabled is False
         assert parser.table_structure is False
         assert parser.export_format == "text"
-        assert parser.use_rapidocr is False
+        assert parser.ocr_engine == "rapidocr"
         assert parser.ocr_languages == ["german"]
         assert parser.force_full_page_ocr is True
 
@@ -90,44 +90,169 @@ class TestDoclingParserInitParser:
 
 
 @pytest.mark.unit
-class TestGetOCROptions:
+class TestOcrEngineSelection:
+    """``OCR_ENGINE`` resolution and per-engine option building."""
 
-    def test_returns_none_when_rapidocr_disabled(self):
+    @pytest.fixture
+    def settings(self):
+        from application.core.settings import settings
+
+        return settings
+
+    def test_default_setting_is_tesseract(self, settings):
+        assert settings.OCR_ENGINE == "tesseract"
+        assert settings.OCR_LANGS == "eng"
+
+    def test_none_reads_setting(self, settings, monkeypatch):
+        from application.parser.file.docling_parser import _resolve_ocr_engine
+
+        monkeypatch.setattr(settings, "OCR_ENGINE", "auto")
+        assert _resolve_ocr_engine(None) == "auto"
+
+    def test_unknown_engine_degrades_to_auto(self):
+        from application.parser.file.docling_parser import _resolve_ocr_engine
+
+        assert _resolve_ocr_engine("easyocr") == "auto"
+
+    def test_tesseract_without_binary_degrades_to_auto(self, monkeypatch):
+        import application.parser.file.docling_parser as dp
+
+        monkeypatch.setattr(dp.shutil, "which", lambda name: None)
+        assert dp._resolve_ocr_engine("tesseract") == "auto"
+
+    def test_tesseract_with_binary_selected(self, monkeypatch):
+        import application.parser.file.docling_parser as dp
+
+        monkeypatch.setattr(dp.shutil, "which", lambda name: "/usr/bin/tesseract")
+        assert dp._resolve_ocr_engine("tesseract") == "tesseract"
+
+    def test_ocrmac_off_darwin_degrades_to_auto(self, monkeypatch):
+        import application.parser.file.docling_parser as dp
+
+        monkeypatch.setattr(dp.sys, "platform", "linux")
+        assert dp._resolve_ocr_engine("ocrmac") == "auto"
+
+    def test_rapidocr_missing_degrades_to_auto(self, monkeypatch):
+        import sys
+
+        import application.parser.file.docling_parser as dp
+
+        monkeypatch.setitem(sys.modules, "rapidocr", None)
+        assert dp._resolve_ocr_engine("rapidocr") == "auto"
+
+    def test_deepseek_passes_through(self):
+        from application.parser.file.docling_parser import _resolve_ocr_engine
+
+        assert _resolve_ocr_engine("deepseek") == "deepseek"
+
+    def test_build_auto_returns_none(self):
+        from application.parser.file.docling_parser import _build_ocr_options
+
+        assert _build_ocr_options("auto", None, True) is None
+
+    def test_build_tesseract_reads_ocr_langs(self, settings, monkeypatch):
+        pytest.importorskip("docling")
+        from application.parser.file.docling_parser import _build_ocr_options
+
+        monkeypatch.setattr(settings, "OCR_LANGS", "eng+chi_sim")
+        options = _build_ocr_options("tesseract", None, True)
+
+        assert type(options).__name__ == "TesseractCliOcrOptions"
+        assert options.lang == ["eng", "chi_sim"]
+        assert options.force_full_page_ocr is True
+
+    def test_build_tesseract_explicit_languages_win(self):
+        pytest.importorskip("docling")
+        from application.parser.file.docling_parser import _build_ocr_options
+
+        options = _build_ocr_options("tesseract", ["deu"], False)
+        assert options.lang == ["deu"]
+
+    def test_build_rapidocr(self):
+        pytest.importorskip("docling")
+        from application.parser.file.docling_parser import _build_ocr_options
+
+        options = _build_ocr_options("rapidocr", None, False)
+        assert type(options).__name__ == "RapidOcrOptions"
+        assert options.lang == ["english"]
+
+    def test_build_import_failure_returns_none(self, monkeypatch):
+        import sys
+
+        from application.parser.file.docling_parser import _build_ocr_options
+
+        monkeypatch.setitem(sys.modules, "docling.datamodel.pipeline_options", None)
+        assert _build_ocr_options("tesseract", ["eng"], False) is None
+        assert _build_ocr_options("ocrmac", None, False) is None
+
+    def test_build_generic_failure_returns_none(self, monkeypatch):
+        import sys
+        import types
+
+        from application.parser.file.docling_parser import _build_ocr_options
+
+        fake = types.ModuleType("docling.datamodel.pipeline_options")
+
+        def _boom(**kwargs):
+            raise RuntimeError("bad options")
+
+        fake.RapidOcrOptions = _boom
+        monkeypatch.setitem(sys.modules, "docling.datamodel.pipeline_options", fake)
+        assert _build_ocr_options("rapidocr", None, False) is None
+
+
+@pytest.mark.unit
+class TestDeepseekVlmConverter:
+    """OCR_ENGINE=deepseek swaps the whole pipeline for docling's VLM route."""
+
+    @pytest.fixture(autouse=True)
+    def _requires_docling(self):
+        pytest.importorskip("docling")
+
+    def test_deepseek_builds_vlm_converter(self, monkeypatch):
+        from application.core.settings import settings
         from application.parser.file.docling_parser import DoclingParser
 
-        parser = DoclingParser(use_rapidocr=False)
-        assert parser._get_ocr_options() is None
+        monkeypatch.setattr(settings, "OCR_DEEPSEEK_URL", "http://gpu-host:8000/v1/chat/completions")
+        monkeypatch.setattr(settings, "OCR_DEEPSEEK_MODEL", "deepseek-ocr-x")
 
-    def test_returns_options_when_available(self):
+        built = {}
+
+        def _capture_converter(format_options):
+            built["format_options"] = format_options
+            return MagicMock()
+
+        monkeypatch.setattr(
+            "docling.document_converter.DocumentConverter", _capture_converter
+        )
+        parser = DoclingParser(ocr_enabled=True, ocr_engine="deepseek")
+        parser._create_converter()
+
+        from docling.datamodel.base_models import InputFormat
+        from docling.pipeline.vlm_pipeline import VlmPipeline
+
+        pdf_option = built["format_options"][InputFormat.PDF]
+        image_option = built["format_options"][InputFormat.IMAGE]
+        assert pdf_option.pipeline_cls is VlmPipeline
+        assert image_option.pipeline_cls is VlmPipeline
+        vlm = pdf_option.pipeline_options.vlm_options
+        assert vlm.url == "http://gpu-host:8000/v1/chat/completions"
+        assert vlm.params["model"] == "deepseek-ocr-x"
+        assert pdf_option.pipeline_options.enable_remote_services is True
+
+    def test_deepseek_ignored_when_ocr_disabled(self, monkeypatch):
         from application.parser.file.docling_parser import DoclingParser
 
-        parser = DoclingParser(use_rapidocr=True, ocr_languages=["english"])
+        vlm_called = []
+        monkeypatch.setattr(
+            DoclingParser,
+            "_create_vlm_converter",
+            lambda self: vlm_called.append(True),
+        )
+        monkeypatch.setattr("docling.document_converter.DocumentConverter", MagicMock())
+        DoclingParser(ocr_enabled=False, ocr_engine="deepseek")._create_converter()
 
-        mock_options = MagicMock()
-        with patch(
-            "application.parser.file.docling_parser.DoclingParser._get_ocr_options",
-            return_value=mock_options,
-        ):
-            result = parser._get_ocr_options()
-            assert result is mock_options
-
-    def test_returns_none_on_import_error(self):
-        from application.parser.file.docling_parser import DoclingParser
-
-        parser = DoclingParser(use_rapidocr=True)
-
-        # Simulate the ImportError path
-        original = parser._get_ocr_options
-
-        def patched_get_ocr():
-            try:
-                raise ImportError("No RapidOcrOptions")
-            except ImportError:
-                return None
-
-        parser._get_ocr_options = patched_get_ocr
-        assert parser._get_ocr_options() is None
-        parser._get_ocr_options = original
+        assert vlm_called == []
 
 
 # =====================================================================
@@ -431,31 +556,6 @@ class TestDoclingSubclasses:
 
 @pytest.mark.unit
 class TestDoclingParserGaps:
-    def test_get_ocr_options_import_error_returns_none(self):
-        """Cover lines 148-150: ImportError returns None."""
-        from application.parser.file.docling_parser import DoclingParser
-
-        parser = DoclingParser(ocr_enabled=True, use_rapidocr=True)
-        with patch.dict("sys.modules", {"docling.datamodel.pipeline_options": None}):
-            # Force re-import to trigger ImportError
-            with patch(
-                "builtins.__import__", side_effect=ImportError("no module")
-            ):
-                result = parser._get_ocr_options()
-                assert result is None
-
-    def test_get_ocr_options_generic_error_returns_none(self):
-        """Cover lines 151-153: generic Exception returns None."""
-        from application.parser.file.docling_parser import DoclingParser
-
-        parser = DoclingParser(ocr_enabled=True, use_rapidocr=True)
-        with patch(
-            "builtins.__import__",
-            side_effect=RuntimeError("unexpected"),
-        ):
-            result = parser._get_ocr_options()
-            assert result is None
-
     def test_csv_parser_init(self):
         """Cover line 289: DoclingCSVParser.__init__ calls super."""
         from application.parser.file.docling_parser import DoclingCSVParser
@@ -1442,11 +1542,11 @@ class TestForceFullPageOCRWiring:
         DoclingParser(**kwargs)._create_converter()
         return built[0]
 
-    def test_forced_without_rapidocr(self, monkeypatch):
+    def test_forced_with_default_auto_options(self, monkeypatch):
         options = self._pipeline_options(
             monkeypatch,
             ocr_enabled=True,
-            use_rapidocr=False,
+            ocr_engine="auto",
             force_full_page_ocr=True,
         )
         assert options.ocr_options.force_full_page_ocr is True
@@ -1455,7 +1555,7 @@ class TestForceFullPageOCRWiring:
         options = self._pipeline_options(
             monkeypatch,
             ocr_enabled=True,
-            use_rapidocr=False,
+            ocr_engine="auto",
             force_full_page_ocr=False,
         )
         assert options.ocr_options.force_full_page_ocr is False

tests/parser/file/test_pdf_trust.py

@@ -0,0 +1,148 @@
+"""Tests for the PDF trust checks behind ``PDF_TRUST_CHECK``.
+
+The checks target the two classes where anydoc drops text *silently*:
+composite (Type0) fonts without a ToUnicode map, and CJK-declaring PDFs
+whose extracted text carries almost no CJK. The scanner is byte-level and
+regex-based, so the synthetic fixtures below are minimal PDF fragments, not
+well-formed files; the one real fixture is the corpus document that loses
+its whole Chinese column in anydoc with no error.
+"""
+import zlib
+from pathlib import Path
+
+from application.parser.file.pdf_trust import (
+    check_pdf_fonts,
+    verify_extraction,
+    verify_pdf_file,
+)
+
+FIXTURES = Path(__file__).parent / "fixtures"
+
+
+def _obj(body: bytes) -> bytes:
+    return b"1 0 obj" + body + b"endobj\n"
+
+
+def _stream(payload: bytes) -> bytes:
+    return b"2 0 obj<</Filter/FlateDecode>>stream\n" + zlib.compress(payload) + b"\nendstream endobj\n"
+
+
+TYPE0_WITH_TOUNICODE = _obj(
+    b"<</Type/Font/Subtype/Type0/BaseFont/AAAAAA+Simple/ToUnicode 9 0 R>>"
+)
+TYPE0_BARE = _obj(b"<</Type/Font/Subtype/Type0/BaseFont/AAAAAA+Bare>>")
+SIMPLE_FONT = _obj(b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>")
+GB1_ORDERING = _obj(b"<</Registry(Adobe)/Ordering(GB1)/Supplement 5>>")
+
+
+class TestCheckPdfFonts:
+    def test_type0_with_tounicode_not_flagged(self):
+        result = check_pdf_fonts(TYPE0_WITH_TOUNICODE)
+        assert result["type0"] == 1
+        assert result["type0_no_tounicode"] == 0
+        assert not result["flagged"]
+
+    def test_type0_without_tounicode_flagged(self):
+        result = check_pdf_fonts(TYPE0_BARE)
+        assert result["type0_no_tounicode"] == 1
+        assert result["flagged"]
+
+    def test_simple_font_not_flagged(self):
+        result = check_pdf_fonts(SIMPLE_FONT)
+        assert result["type0"] == 0
+        assert result["has_fonts"]
+        assert not result["flagged"]
+
+    def test_no_fonts_flagged(self):
+        result = check_pdf_fonts(_obj(b"<</Type/Page>>"))
+        assert not result["has_fonts"]
+        assert result["flagged"]
+
+    def test_type0_inside_object_stream_is_seen(self):
+        """Object streams hold dicts with no obj markers; the scan must decompress them."""
+        data = _stream(b"<</Type/Font/Subtype/Type0/BaseFont/BBBBBB+Packed>>")
+        result = check_pdf_fonts(data)
+        assert result["type0"] == 1
+        assert result["type0_no_tounicode"] == 1
+        assert result["flagged"]
+
+    def test_font_inside_object_stream_counts_as_has_fonts(self):
+        """A fully-compressed PDF with only simple fonts must not false-positive."""
+        data = _stream(b"<</Type/Font/Subtype/TrueType/BaseFont/Arial>>")
+        result = check_pdf_fonts(data)
+        assert result["has_fonts"]
+        assert not result["flagged"]
+
+    def test_cjk_ordering_detected_in_stream(self):
+        data = _stream(b"<</Registry(Adobe)/Ordering(CNS1)/Supplement 4>>") + SIMPLE_FONT
+        assert check_pdf_fonts(data)["expects_cjk"]
+
+    def test_cjk_font_names_alone_do_not_set_expectation(self):
+        """Regression for the Berkshire false positive: Word-exported Latin PDFs
+        embed MS-Gothic subsets for stray full-width characters, and substring
+        matching also hits FranklinGothic — neither means CJK *content*, and a
+        false expectation reroutes a 150-page English report to the heavy
+        engine. Only CIDSystemInfo /Ordering may set the expectation."""
+        data = _obj(
+            b"<</Type/Font/Subtype/TrueType/BaseFont/AKBHWD+MS-Gothic>>"
+        ) + _obj(b"<</Type/Font/Subtype/TrueType/BaseFont/KMBGKH+FranklinGothic-Roman>>")
+        result = check_pdf_fonts(data)
+        assert not result["expects_cjk"]
+        assert not result["flagged"]
+
+
+class TestVerifyExtraction:
+    def test_clean_pdf_and_output(self):
+        assert verify_extraction(SIMPLE_FONT, "Plain English text.") == []
+
+    def test_bare_type0_reported(self):
+        problems = verify_extraction(TYPE0_BARE + SIMPLE_FONT, "some text")
+        assert len(problems) == 1
+        assert "ToUnicode" in problems[0]
+
+    def test_cjk_expected_but_missing(self):
+        problems = verify_extraction(GB1_ORDERING + SIMPLE_FONT, "English only output")
+        assert len(problems) == 1
+        assert "CJK" in problems[0]
+
+    def test_cjk_expected_and_present(self):
+        text = "标题：本协议由双方共同签署生效" # 14 CJK chars
+        assert verify_extraction(GB1_ORDERING + SIMPLE_FONT, text) == []
+
+    def test_both_problems_reported(self):
+        problems = verify_extraction(TYPE0_BARE + GB1_ORDERING, "english")
+        assert len(problems) == 2
+
+
+class TestRealFixture:
+    """The corpus PDF where anydoc silently drops the entire Chinese column."""
+
+    def test_cid_font_nda_is_flagged(self):
+        data = (FIXTURES / "nda_en_zh_cid_font.pdf").read_bytes()
+        result = check_pdf_fonts(data)
+        assert result["type0_no_tounicode"] >= 1
+        assert result["expects_cjk"]
+        assert result["flagged"]
+
+    def test_cjkless_extraction_reports_both(self):
+        data = (FIXTURES / "nda_en_zh_cid_font.pdf").read_bytes()
+        problems = verify_extraction(data, "MUTUAL NON-DISCLOSURE AGREEMENT ...")
+        assert any("ToUnicode" in p for p in problems)
+        assert any("CJK" in p for p in problems)
+
+
+def test_flate_bomb_streams_are_capped():
+    """Flate reaches ~1000:1, so per-stream inflation must be capped — an
+    uncapped decompress of a crafted PDF would OOM the ingest worker."""
+    from application.parser.file.pdf_trust import _decompressed_streams, _STREAM_INFLATE_CAP
+
+    bomb = _stream(b"\0" * (_STREAM_INFLATE_CAP * 4))
+    chunks = list(_decompressed_streams(bomb))
+    assert chunks
+    assert all(len(chunk) <= _STREAM_INFLATE_CAP for chunk in chunks)
+    # The capped scan still completes and reports sanely alongside real objects.
+    assert not check_pdf_fonts(SIMPLE_FONT + bomb)["flagged"]
+
+
+def test_verify_pdf_file_unreadable_trusts_output(tmp_path):
+    assert verify_pdf_file(tmp_path / "missing.pdf", "text") == []

tests/parser/file/test_tableize.py

@@ -0,0 +1,74 @@
+"""Tests for the dot-leader/whitespace table reconstruction (``ANYDOC_TABLEIZE``)."""
+from application.parser.file.tableize import tableize
+
+DOT_LEADER = """Revenues
+Insurance premiums ............ 83,431 77,731
+Sales and service revenues ........ 24,660 23,406
+Freight rail transportation ......... 22,341 23,852
+Total revenues 130,432 124,989
+"""
+
+
+def test_dot_leader_run_becomes_table():
+    out = tableize(DOT_LEADER)
+    assert "| Insurance premiums | 83,431 | 77,731 |" in out
+    assert "| Total revenues | 130,432 | 124,989 |" in out
+    assert "| --- |" in out
+    assert "Revenues" in out  # heading line untouched
+
+
+def test_whitespace_only_rows_convert_too():
+    md = "Alpha 1 2\nBeta 3 4\nGamma 5 6\n"
+    out = tableize(md)
+    assert "| Alpha | 1 | 2 |" in out
+
+
+def test_currency_symbol_merges_with_number():
+    md = "Cash $ 1,234 900\nDebt $ 2,000 1,500\nEquity $ 900 800\n"
+    out = tableize(md)
+    assert "| Cash | $1,234 | 900 |" in out
+
+
+def test_parenthesised_negatives_and_dash():
+    md = "Losses (30) —\nGains (12) —\nNet (42) —\n"
+    out = tableize(md)
+    assert "| Losses | (30) | — |" in out
+
+
+def test_short_run_is_left_alone():
+    md = "Alpha 1 2\nBeta 3 4\n"
+    assert "|" not in tableize(md)
+
+
+def test_mixed_widths_are_left_alone():
+    md = "Alpha 1 2\nBeta 3\nGamma 5 6\n"
+    assert "|" not in tableize(md)
+
+
+def test_prose_with_numbers_is_left_alone():
+    md = "The division shipped 47 releases in 2023.\nIt hired 12 engineers this year alone.\nRevenue grew by a factor of 3 since 2019.\n"
+    assert "|" not in tableize(md)
+
+
+def test_min_rows_boundary():
+    two = "A 1 2\nB 3 4\n"
+    three = two + "C 5 6\n"
+    assert "|" not in tableize(two, min_rows=3)
+    assert "|" in tableize(three, min_rows=3)
+
+
+def test_pipe_in_label_is_escaped():
+    md = "Assets | current 1,234 900\nDebt 2,000 1,500\nEquity 900 800\n"
+    out = tableize(md)
+    assert "| Assets \\| current | 1,234 | 900 |" in out
+
+
+def test_dollar_prefixed_word_is_not_a_value():
+    md = "Alpha $TBD 2\nBeta $TBD 4\nGamma $TBD 6\n"
+    assert "|" not in tableize(md)
+
+
+def test_non_table_text_passes_through_verbatim():
+    """Only the trailing newline may differ (splitlines/join round-trip)."""
+    md = "# Heading\n\nA paragraph with no numbers.\n\n- a list item\n"
+    assert tableize(md) == md.rstrip("\n")