Fixes batch 1

tests/parser/file/test_anydoc_parser.py

@@ -534,3 +534,81 @@ def test_non_scan_refusals_do_not_trigger_the_guard(tmp_path, monkeypatch):
 
     out = AnydocParser(fallback_parser=_RecordingFallback(result="tiny")).parse_file(path)
     assert out == "tiny"
+
+
+# --- spreadsheets: a resource limit delegates, other formats stay terminal ----
+
+
+def test_resource_limit_on_a_spreadsheet_delegates_to_the_tabular_fallback(tmp_path, monkeypatch):
+    """anydoc refuses a 265k-row sheet on fixed limits that openpyxl/pandas
+    read fine, so for tabular suffixes the limit is a reason to fall back."""
+    fake = _fake_anydoc(monkeypatch, None)
+
+    def _refuse(path):
+        raise fake.ResourceLimitError("resource limit exceeded (max_xml_nodes): 2000000")
+
+    fake.to_markdown = _refuse
+    fallback = _RecordingFallback()
+    parser = AnydocParser(fallback_parser=fallback)
+    path = tmp_path / "big.xlsx"
+    path.write_bytes(b"PK")
+
+    assert parser.parse_file(path) == FALLBACK_TEXT
+    assert fallback.calls == [path]
+    assert parser.last_engine == "_RecordingFallback"
+
+
+def test_resource_limit_on_a_spreadsheet_without_fallback_is_terminal(tmp_path, monkeypatch):
+    fake = _fake_anydoc(monkeypatch, None)
+
+    def _refuse(path):
+        raise fake.ResourceLimitError("resource limit exceeded (max_entry_bytes)")
+
+    fake.to_markdown = _refuse
+    path = tmp_path / "big.xlsx"
+    path.write_bytes(b"PK")
+
+    with pytest.raises(DocumentParseError, match="max_entry_bytes"):
+        AnydocParser().parse_file(path)
+
+
+# --- mixed documents: one page's OCR failure costs that page only ------------
+
+
+class _PerPageOcrFallback(BaseParser):
+    ocr_enabled = True
+
+    def __init__(self, failing_index):
+        super().__init__()
+        self.failing_index = failing_index
+        self.requests = []
+
+    def _init_parser(self):
+        return {}
+
+    def parse_file(self, file, errors="ignore"):
+        return "unused"
+
+    def ocr_pages(self, file, indices):
+        self.requests.append(list(indices))
+        if self.failing_index in indices:
+            raise DocumentParseError("tesseract failed on this page")
+        return {index: f"OCR TEXT PAGE {index + 1}" for index in indices}
+
+
+def test_one_failing_page_does_not_drop_the_other_pages_ocr(tmp_path, monkeypatch):
+    pytest.importorskip("pypdfium2")
+    pytest.importorskip("pypdf")
+    _fake_anydoc(monkeypatch, lambda path: "Text pages read by anydoc.")
+    fallback = _PerPageOcrFallback(failing_index=0)
+    parser = AnydocParser(fallback_parser=fallback)
+    path = _scanned_pdf(tmp_path / "mixed.pdf", pages=3)  # every page probes as scanned
+
+    text = parser.parse_file(path)
+
+    assert fallback.requests == [[0], [1], [2]]
+    assert "OCR TEXT PAGE 1" not in text
+    assert "OCR TEXT PAGE 2" in text and "OCR TEXT PAGE 3" in text
+    assert text.startswith("Text pages read by anydoc.")
+    # (The font-less blank-page fixture also earns a trust-check warning.)
+    assert parser.get_file_metadata(path)["ocr_pages"] == 2

tests/parser/file/test_docling_parser.py

@@ -568,7 +568,51 @@ class TestDoclingParserGaps:
 
         parser = DoclingCSVParser()
         assert parser.export_format == "markdown"
-        assert parser.ocr_enabled is True
+        assert parser.ocr_enabled is False
+
+
+@pytest.mark.unit
+class TestNonOcrParsersLeaveTextAlone:
+    """Office/markup docling parsers never OCR, so the tesseract CJK
+    post-processing must not touch their born-digital exports."""
+
+    @pytest.mark.parametrize(
+        "name",
+        [
+            "DoclingDocxParser",
+            "DoclingPPTXParser",
+            "DoclingXLSXParser",
+            "DoclingHTMLParser",
+            "DoclingCSVParser",
+            "DoclingMarkdownParser",
+            "DoclingAsciiDocParser",
+            "DoclingVTTParser",
+            "DoclingXMLParser",
+        ],
+    )
+    def test_ocr_is_off_by_construction(self, name):
+        import application.parser.file.docling_parser as dp
+
+        assert getattr(dp, name)().ocr_enabled is False
+
+    def test_cjk_spaces_survive_in_a_docx_export(self, monkeypatch):
+        from application.core.settings import settings
+        from application.parser.file.docling_parser import DoclingDocxParser
+
+        monkeypatch.setattr(settings, "OCR_ENGINE", "tesseract")
+        parser = DoclingDocxParser()
+        # Simulate what _create_converter records for an OCR-on parser; the
+        # docx parser is OCR-off so the collapse must stay inert regardless.
+        parser._active_ocr_engine = "tesseract"
+        text = "\u5f20 \u4e09 \u548c \u674e \u56db\n\uff21\uff22\uff23 \uff24\uff25\uff26"
+        assert parser._postprocess_ocr_text(text) == text
+
+    def test_pdf_parser_with_tesseract_still_collapses(self):
+        from application.parser.file.docling_parser import DoclingPDFParser
+
+        parser = DoclingPDFParser(ocr_enabled=True)
+        parser._active_ocr_engine = "tesseract"
+        assert parser._postprocess_ocr_text("\u4e92\u76f8 \u4fdd\u5bc6") == "\u4e92\u76f8\u4fdd\u5bc6"
 
 
 # =====================================================================
@@ -1331,11 +1375,11 @@ class TestOCRDropoutGuard:
         assert create.call_count == 1
 
     def test_guard_does_not_apply_to_non_ocr_formats(self):
-        """DOCX parsers inherit ocr_enabled=True but never OCR anything."""
+        """DOCX parsers construct with OCR off: they never OCR anything."""
         from application.parser.file.docling_parser import DoclingDocxParser
 
         parser = DoclingDocxParser()
-        assert parser.ocr_enabled is True
+        assert parser.ocr_enabled is False
         converter = MagicMock()
         converter.convert.return_value = _mock_conversion("Hi", pages=1)
         parser._converter = converter

tests/parser/file/test_ocr_parser.py

@@ -10,6 +10,7 @@ import sys
 from pathlib import Path
 from unittest.mock import MagicMock, patch
 
+import logging
 import pytest
 
 pytest.importorskip("pypdfium2")
@@ -748,7 +749,7 @@ class TestAnydocMixedDocuments:
             content = parser.parse_file(mixed_pdf)
 
         assert "Digital text page" in content
-        assert "text pages only" in caplog.text
+        assert "skipping that page" in caplog.text
         assert parser.get_file_metadata(mixed_pdf) == {}
 
     def test_all_text_document_never_probes_ocr(self, tmp_path):
@@ -761,3 +762,88 @@ class TestAnydocMixedDocuments:
         parser = AnydocParser(fallback_parser=fallback)
         parser.parse_file(pdf)
         fallback.ocr_pages.assert_not_called()
+
+
+# ---------------------------------------------------------------------------
+# Render budget: an oversized MediaBox must not become a multi-GB bitmap
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.unit
+class TestRenderBudget:
+    def test_letter_page_is_not_clamped(self):
+        assert op.render_scale(612, 792, 200) == pytest.approx(200 / 72)
+
+    def test_pdf_maximum_page_is_clamped_to_the_budget(self):
+        scale = op.render_scale(14400, 14400, 200)
+        assert scale < 200 / 72
+        assert (14400 * scale) * (14400 * scale) <= op._MAX_RENDER_PIXELS * 1.001
+
+    def test_huge_page_renders_within_budget(self, tmp_path, caplog):
+        """A 14400x14400 pt blank page (the PDF maximum, a few hundred bytes)
+        would be 40000x40000 px = 6 GiB at 200 dpi without the cap."""
+        pytest.importorskip("pypdf")
+        import pypdfium2 as pdfium
+        from pypdf import PdfWriter
+
+        path = tmp_path / "huge.pdf"
+        writer = PdfWriter()
+        writer.add_blank_page(width=14400, height=14400)
+        with open(path, "wb") as fh:
+            writer.write(fh)
+
+        pdf = pdfium.PdfDocument(str(path))
+        try:
+            page = pdf[0]
+            try:
+                with caplog.at_level(logging.WARNING, logger="application.parser.file.ocr_parser"):
+                    image = op._render_page(page, 200)
+            finally:
+                page.close()
+        finally:
+            pdf.close()
+        assert image.width * image.height <= op._MAX_RENDER_PIXELS * 1.001
+        assert "stay within" in caplog.text
+
+
+# ---------------------------------------------------------------------------
+# Scanned-page probe: thin text layers count only over an image
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.unit
+class TestScannedPageProbeImageGate:
+    def _cover_and_figure_pdf(self, path: Path) -> Path:
+        pytest.importorskip("reportlab")
+        from reportlab.lib.pagesizes import letter
+        from reportlab.lib.utils import ImageReader
+        from reportlab.pdfgen import canvas
+
+        c = canvas.Canvas(str(path), pagesize=letter)
+        # Page 1: a short title and nothing else — not a scan, nothing to OCR.
+        c.setFont("Helvetica", 36)
+        c.drawString(72, 500, "Annual Report 2025")
+        c.showPage()
+        # Page 2: a page number under a full-page raster figure — a scan with a stamp.
+        img = Image.new("RGB", (300, 400), "white")
+        ImageDraw.Draw(img).rectangle((50, 50, 250, 350), outline="black", width=3)
+        c.drawImage(ImageReader(img), 72, 150, width=450, height=600)
+        c.setFont("Helvetica", 10)
+        c.drawString(300, 60, "17")
+        c.showPage()
+        # Page 3: ordinary text.
+        c.setFont("Helvetica", 12)
+        c.drawString(72, 700, "A full paragraph of ordinary body text that anydoc reads directly.")
+        c.showPage()
+        c.save()
+        return path
+
+    def test_short_text_page_without_image_is_not_scanned(self, tmp_path):
+        path = self._cover_and_figure_pdf(tmp_path / "cover.pdf")
+        counts = op.text_layer_counts(path)
+        assert 0 < counts[0] < op.DEFAULT_TEXT_LAYER_MIN_CHARS
+        assert 0 < counts[1] < op.DEFAULT_TEXT_LAYER_MIN_CHARS
+        assert op.scanned_page_indices(path) == [1]
+
+    def test_pages_with_no_text_at_all_still_count(self, tmp_path):
+        assert op.scanned_page_indices(_image_pdf(tmp_path / "scan.pdf", pages=2)) == [0, 1]

tests/parser/file/test_pdf_trust.py

@@ -146,3 +146,79 @@ def test_flate_bomb_streams_are_capped():
 
 def test_verify_pdf_file_unreadable_trusts_output(tmp_path):
     assert verify_pdf_file(tmp_path / "missing.pdf", "text") == []
+
+
+# --- stream scanning is a keyword walk, not a regex -----------------------------
+
+
+class TestStreamWalk:
+    def test_stream_without_eol_before_endstream_is_seen(self):
+        body = zlib.compress(b"<</Type/Font/Subtype/Type0/BaseFont/X>>")
+        data = b"2 0 obj<</Filter/FlateDecode>>stream\n" + body + b"endstream endobj\n"
+        assert check_pdf_fonts(data)["type0"] == 1
+
+    def test_undecodable_stream_does_not_swallow_the_next_one(self):
+        """A non-Flate (image) stream before a font stream must not merge the two."""
+        image = b"3 0 obj<</Filter/DCTDecode>>stream\n\xff\xd8 not zlib at all\nendstream endobj\n"
+        font = _stream(b"<</Type/Font/Subtype/Type0/BaseFont/X/ToUnicode 9 0 R>>")
+        result = check_pdf_fonts(image + font)
+        assert result["type0"] == 1
+        assert result["type0_no_tounicode"] == 0
+
+    def test_crlf_streams_decompress(self):
+        body = zlib.compress(b"/Ordering (GB1)")
+        data = b"2 0 obj<</Filter/FlateDecode>>stream\r\n" + body + b"\r\nendstream endobj\n"
+        assert check_pdf_fonts(data)["expects_cjk"] is True
+
+    def test_large_file_with_many_streams_is_linear(self):
+        """1000 undecodable streams: the old regex went quadratic here."""
+        import time
+
+        chunk = b"9 0 obj<</Length 30>>stream\n" + b"\x00" * 30 + b"endstream endobj\n"
+        data = chunk * 1000 + TYPE0_BARE
+        started = time.perf_counter()
+        result = check_pdf_fonts(data)
+        assert time.perf_counter() - started < 1.0
+        assert result["type0_no_tounicode"] == 1
+
+
+# --- the CJK cross-check is confirmed against pdfium's text layer --------------
+
+
+def _latin_pdf(path: Path) -> Path:
+    import pytest
+
+    pytest.importorskip("reportlab")
+    from reportlab.pdfgen import canvas
+
+    c = canvas.Canvas(str(path))
+    c.drawString(72, 700, "Plain English content, nothing else on the page.")
+    c.showPage()
+    c.save()
+    return path
+
+
+def test_stray_cjk_ordering_without_cjk_text_is_not_flagged(tmp_path):
+    """A Quartz-style export declares Japan1 on one unused font: pdfium sees no
+    CJK either, so anydoc dropped nothing and the file must not be rerouted."""
+    import pytest
+
+    pytest.importorskip("pypdfium2")
+    path = _latin_pdf(tmp_path / "latin.pdf")
+    with open(path, "ab") as fh:
+        fh.write(b"\n% trailing junk\n" + _obj(b"<</Registry(Adobe)/Ordering(Japan1)/Supplement 6>>"))
+    markdown = "Plain English content, nothing else on the page."
+
+    # The byte-level check alone still fires...
+    assert any(p.startswith("PDF declares CJK fonts") for p in verify_extraction(path.read_bytes(), markdown))
+    # ...and the file-level check withdraws it.
+    assert [p for p in verify_pdf_file(path, markdown) if p.startswith("PDF declares CJK fonts")] == []
+
+
+def test_real_cjk_drop_survives_the_cross_check():
+    import pytest
+
+    pytest.importorskip("pypdfium2")
+    fixture = FIXTURES / "nda_en_zh_cid_font.pdf"
+    problems = verify_pdf_file(fixture, "English only, the Chinese column is gone.")
+    assert any(p.startswith("PDF declares CJK fonts") for p in problems)

tests/parser/file/test_pdfium_parser.py

@@ -165,6 +165,9 @@ def docling_engine(monkeypatch):
     from application.core.settings import settings
 
     monkeypatch.setattr(settings, "DOC_PARSER_ENGINE", "docling")
+    # A developer .env with OCR on (and OCR_BACKEND=native) would otherwise
+    # swap the native OCR parser in for the docling one these tests expect.
+    monkeypatch.setattr(settings, "OCR_ENABLED", False)
 
 
 def test_extractor_defaults_to_docling_for_pdf(docling_engine):