Batch fixes 2

tests/parser/file/test_anydoc_parser.py

@@ -308,7 +308,7 @@ def test_init_parser_imports_for_real_not_just_find_spec(monkeypatch):
     """A wheel whose native extension fails to load has a spec but no importable
     module; that must surface at init, not as a bare ImportError from parse_file
     mid-ingest (which load_data does not catch)."""
-    import application.parser.file.anydoc_parser as mod
+    from application.parser.file import anydoc_parser as mod
 
     monkeypatch.setattr(mod, "anydoc_available", lambda: True)
     monkeypatch.setitem(sys.modules, "anydoc", None)
@@ -337,6 +337,7 @@ class _FakeDoclingFallback:
 
         class _Inner(DoclingParser):
             def __init__(self):
+                super().__init__(ocr_enabled=False)
                 self._parser_config = {}
                 self.calls = []
                 self.last_engine = None

tests/parser/file/test_bulk.py

@@ -749,3 +749,16 @@ class TestGainedFormats:
 
         assert "# Title" in out
         assert "Body text here." in out
+
+
+def test_gained_suffix_without_a_parser_is_rejected_not_read_as_text(tmp_path):
+    """Without anydoc an OLE .doc must not be indexed as decoded binary garbage."""
+    from application.parser.file.base_parser import DocumentParseError
+    from application.parser.file.bulk import SimpleDirectoryReader
+
+    path = tmp_path / "legacy.doc"
+    path.write_bytes(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 64)
+    reader = SimpleDirectoryReader(input_files=[str(path)], file_extractor={".pdf": object()})
+
+    with pytest.raises(DocumentParseError, match="firecrawl-anydoc"):
+        reader.load_data()

tests/parser/file/test_docling_parser.py

@@ -121,19 +121,19 @@ class TestOcrEngineSelection:
         assert _resolve_ocr_engine("easyocr") == "auto"
 
     def test_tesseract_without_binary_degrades_to_auto(self, monkeypatch):
-        import application.parser.file.docling_parser as dp
+        from application.parser.file import docling_parser as dp
 
         monkeypatch.setattr(dp.shutil, "which", lambda name: None)
         assert dp._resolve_ocr_engine("tesseract") == "auto"
 
     def test_tesseract_with_binary_selected(self, monkeypatch):
-        import application.parser.file.docling_parser as dp
+        from application.parser.file import docling_parser as dp
 
         monkeypatch.setattr(dp.shutil, "which", lambda name: "/usr/bin/tesseract")
         assert dp._resolve_ocr_engine("tesseract") == "tesseract"
 
     def test_ocrmac_off_darwin_degrades_to_auto(self, monkeypatch):
-        import application.parser.file.docling_parser as dp
+        from application.parser.file import docling_parser as dp
 
         monkeypatch.setattr(dp.sys, "platform", "linux")
         assert dp._resolve_ocr_engine("ocrmac") == "auto"
@@ -141,7 +141,7 @@ class TestOcrEngineSelection:
     def test_rapidocr_missing_degrades_to_auto(self, monkeypatch):
         import sys
 
-        import application.parser.file.docling_parser as dp
+        from application.parser.file import docling_parser as dp
 
         monkeypatch.setitem(sys.modules, "rapidocr", None)
         assert dp._resolve_ocr_engine("rapidocr") == "auto"
@@ -167,10 +167,13 @@ class TestOcrEngineSelection:
         assert options.lang == ["eng", "chi_sim"]
         assert options.force_full_page_ocr is True
 
-    def test_build_tesseract_explicit_languages_win(self):
+    def test_build_tesseract_explicit_languages_win(self, monkeypatch):
         pytest.importorskip("docling")
+        import application.parser.file.ocr_parser as op
         from application.parser.file.docling_parser import _build_ocr_options
 
+        # Pack inventory unknown: the requested list is passed through untouched.
+        monkeypatch.setattr(op, "tesseract_languages", lambda: None)
         options = _build_ocr_options("tesseract", ["deu"], False)
         assert options.lang == ["deu"]
 
@@ -591,7 +594,7 @@ class TestNonOcrParsersLeaveTextAlone:
         ],
     )
     def test_ocr_is_off_by_construction(self, name):
-        import application.parser.file.docling_parser as dp
+        from application.parser.file import docling_parser as dp
 
         assert getattr(dp, name)().ocr_enabled is False
 
@@ -1673,3 +1676,33 @@ class TestDoclingOcrPages:
         pdf.write_bytes(b"%PDF-1.4")
         with pytest.raises(DocumentParseError, match="page 1 of doc.pdf"):
             parser.ocr_pages(pdf, [0])
+
+
+@pytest.mark.unit
+class TestTesseractLanguageFilter:
+    def test_uninstalled_packs_are_dropped_with_a_warning(self, monkeypatch, caplog):
+        pytest.importorskip("docling")
+        import application.parser.file.ocr_parser as op
+        from application.parser.file.docling_parser import _build_ocr_options
+
+        monkeypatch.setattr(op, "tesseract_languages", lambda: frozenset({"eng", "osd"}))
+        with caplog.at_level("WARNING"):
+            options = _build_ocr_options("tesseract", ["eng", "chi_sim"], False)
+        assert options.lang == ["eng"]
+        assert "chi_sim" in caplog.text
+
+    def test_all_packs_missing_falls_back_to_eng(self, monkeypatch):
+        pytest.importorskip("docling")
+        import application.parser.file.ocr_parser as op
+        from application.parser.file.docling_parser import _build_ocr_options
+
+        monkeypatch.setattr(op, "tesseract_languages", lambda: frozenset({"eng"}))
+        assert _build_ocr_options("tesseract", ["xyz"], False).lang == ["eng"]
+
+    def test_unknown_inventory_keeps_the_list(self, monkeypatch):
+        pytest.importorskip("docling")
+        import application.parser.file.ocr_parser as op
+        from application.parser.file.docling_parser import _build_ocr_options
+
+        monkeypatch.setattr(op, "tesseract_languages", lambda: None)
+        assert _build_ocr_options("tesseract", ["eng", "deu"], False).lang == ["eng", "deu"]

tests/parser/file/test_html_parser.py

@@ -218,7 +218,7 @@ def test_trim_torn_utf8_tail():
 
 def test_markdown_parser_metadata_reuses_last_parse(tmp_path, rich_html_file, monkeypatch):
     """The metadata call after parse_file must not build the soup again."""
-    import application.parser.file.html_parser as mod
+    from application.parser.file import html_parser as mod
 
     parser = HTMLMarkdownParser()
     parser.parse_file(rich_html_file)
@@ -231,3 +231,48 @@ def test_markdown_parser_metadata_reuses_last_parse(tmp_path, rich_html_file, mo
     other.write_text("<title>Other</title>")
     parser.get_file_metadata(other)
     assert len(calls) == 1
+
+
+# --- second-pass fixes: XML prolog, data URIs, UTF-16 heads ---------------------
+
+
+def test_xml_prolog_does_not_leak_into_markdown(tmp_path):
+    from application.parser.file.html_parser import HTMLMarkdownParser
+
+    path = tmp_path / "doc.xhtml"
+    path.write_bytes(
+        b'<?xml version="1.0" encoding="UTF-8"?>\n<!DOCTYPE html>\n'
+        b'<html xmlns="http://www.w3.org/1999/xhtml"><head><title>T</title></head>'
+        b"<body><h1>Hi</h1><p>para</p><![CDATA[raw]]></body></html>"
+    )
+    text = HTMLMarkdownParser().parse_file(path)
+    assert "xml version" not in text
+    assert "raw" not in text
+    assert text.startswith("# Hi")
+
+
+def test_data_uris_are_stripped_from_images_and_links():
+    payload = "data:image/png;base64," + "A" * 200_000
+    html = (
+        f'<p>before</p><img src="{payload}" alt="chart"><a href="{payload}">dl</a>'
+        f'<img srcset="{payload} 1x" alt="x"><p>after</p>'
+    )
+    text = html_to_markdown(html)
+    assert "AAAA" not in text
+    assert "before" in text and "after" in text
+    assert len(text) < 200
+
+
+def test_utf16_head_keeps_an_even_byte_count(tmp_path, monkeypatch):
+    from application.core.settings import settings
+    from application.parser.file.html_parser import HTMLMarkdownParser, read_markup_head
+
+    body = "".join(f"<p>Zeile {i} Über Größe</p>\n" for i in range(200))
+    path = tmp_path / "wide.html"
+    path.write_bytes(("<html><body>" + body + "</body></html>").encode("utf-16"))  # BOM-prefixed
+    head = read_markup_head(path, 3001)
+    assert len(head) % 2 == 0
+    monkeypatch.setattr(settings, "MARKUP_MAX_BYTES", 3001)
+    text = HTMLMarkdownParser().parse_file(path)
+    assert "Zeile 0 Über Größe" in text
+    assert "\x00" not in text

tests/parser/file/test_ocr_parser.py

@@ -10,6 +10,7 @@ import sys
 from pathlib import Path
 from unittest.mock import MagicMock, patch
 
+import io
 import logging
 import pytest
 
@@ -46,7 +47,7 @@ def _text_pdf(
     path: Path, pages: int = 1, text: str = "Hello text layer, plenty of characters here.", lines: int = 1
 ) -> Path:
     """A born-digital PDF; ``lines`` paragraphs per page make anydoc classify it as text-based."""
-    reportlab = pytest.importorskip("reportlab")  # noqa: F841
+    pytest.importorskip("reportlab")
     from reportlab.lib.pagesizes import letter
     from reportlab.pdfgen import canvas
 
@@ -847,3 +848,119 @@ class TestScannedPageProbeImageGate:
 
     def test_pages_with_no_text_at_all_still_count(self, tmp_path):
         assert op.scanned_page_indices(_image_pdf(tmp_path / "scan.pdf", pages=2)) == [0, 1]
+
+
+# ---------------------------------------------------------------------------
+# Second-pass fixes: language packs, alpha, animations, image budget, errors
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.unit
+class TestTesseractLanguagePacks:
+    def test_missing_pack_with_exit_zero_is_a_typed_error(self, monkeypatch):
+        """tesseract drops a missing pack, exits 0 and OCRs with the rest — that must not pass silently."""
+        monkeypatch.setattr(op.shutil, "which", lambda name: "/usr/bin/tesseract")
+
+        def fake_run(cmd, input=None, capture_output=None, timeout=None, check=None):
+            return subprocess.CompletedProcess(
+                cmd, 0, stdout=b"English only\n",
+                stderr=b"Error opening data file /usr/share/tessdata/chi_sim.traineddata\n",
+            )
+
+        monkeypatch.setattr(op.subprocess, "run", fake_run)
+        engine = op.TesseractEngine(languages=["eng", "chi_sim"])
+        with pytest.raises(op.OcrUnavailableError, match="chi_sim"):
+            engine.ocr_image(Image.new("RGB", (10, 10), "white"))
+
+    def test_list_langs_is_parsed(self, monkeypatch):
+        op.tesseract_languages.cache_clear()
+        monkeypatch.setattr(op.shutil, "which", lambda name: "/usr/bin/tesseract")
+
+        def fake_run(cmd, capture_output=None, timeout=None, check=None):
+            return subprocess.CompletedProcess(
+                cmd, 0, stdout=b"List of available languages in /usr/share/tessdata/ (3):\nchi_sim\neng\nosd\n", stderr=b""
+            )
+
+        monkeypatch.setattr(op.subprocess, "run", fake_run)
+        try:
+            assert op.tesseract_languages() == frozenset({"chi_sim", "eng", "osd"})
+        finally:
+            op.tesseract_languages.cache_clear()
+
+
+@pytest.mark.unit
+class TestImageNormalisation:
+    def test_transparent_background_is_flattened_to_white(self):
+        rgba = Image.new("RGBA", (4, 4), (0, 0, 0, 0))
+        rgba.putpixel((1, 1), (0, 0, 0, 255))  # one black "text" pixel
+        flat = Image.open(io.BytesIO(op._png_bytes(rgba)))
+        assert flat.mode == "RGB"
+        assert flat.getpixel((0, 0)) == (255, 255, 255)
+        assert flat.getpixel((1, 1)) == (0, 0, 0)
+
+    def test_palette_transparency_is_flattened(self):
+        pal = Image.new("P", (2, 2), 0)
+        pal.info["transparency"] = 0
+        flat = Image.open(io.BytesIO(op._png_bytes(pal)))
+        assert flat.getpixel((0, 0)) == (255, 255, 255)
+
+    def test_oversized_image_is_downscaled_to_the_budget(self, caplog):
+        big = Image.new("L", (9000, 9000), 255)
+        with caplog.at_level(logging.WARNING, logger="application.parser.file.ocr_parser"):
+            small = op.fit_to_pixel_budget(big)
+        assert small.width * small.height <= op._MAX_RENDER_PIXELS
+        assert "downscaling" in caplog.text
+
+    def test_small_image_is_untouched(self):
+        img = Image.new("RGB", (300, 200), "white")
+        assert op.fit_to_pixel_budget(img) is img
+
+    def test_animated_gif_ocrs_only_the_first_frame(self, tmp_path):
+        frames = [Image.new("RGB", (20, 20), "white") for _ in range(5)]
+        gif = tmp_path / "anim.gif"
+        frames[0].save(gif, save_all=True, append_images=frames[1:], duration=50, loop=0)
+        engine = FakeEngine(["frame text"])
+        parser = op.NativeOcrImageParser(engine=engine)
+        assert parser.parse_file(gif) == "frame text"
+        assert parser.get_file_metadata(gif)["ocr_pages"] == 1
+
+    def test_multi_page_tiff_still_reads_every_frame(self, tmp_path):
+        frames = [Image.new("L", (20, 20), 255) for _ in range(3)]
+        tiff = tmp_path / "fax.tiff"
+        frames[0].save(tiff, save_all=True, append_images=frames[1:])
+        parser = op.NativeOcrImageParser(engine=FakeEngine(["a", "b", "c"]))
+        assert parser.parse_file(tiff) == "a\n\nb\n\nc"
+
+
+@pytest.mark.unit
+class TestDeepseekErrorShapes:
+    def test_non_json_body_is_reported_as_such(self, monkeypatch):
+        import requests
+
+        class _Resp:
+            def raise_for_status(self):
+                return None
+
+            def json(self):
+                raise requests.exceptions.JSONDecodeError("Expecting value", "<html>", 0)
+
+        monkeypatch.setattr(requests, "post", lambda *a, **k: _Resp())
+        engine = op.DeepseekOcrEngine(url="http://proxy.local/v1/chat/completions", model="m")
+        with pytest.raises(DocumentParseError, match="non-JSON body"):
+            engine.ocr_image(Image.new("RGB", (5, 5), "white"))
+
+
+@pytest.mark.unit
+def test_delegate_parse_lets_setup_errors_through():
+    """A fallback whose dependency is missing is a deployment problem, not a bad file."""
+    from application.parser.file.base_parser import BaseParser, delegate_parse
+
+    class _NeedsLib(BaseParser):
+        def _init_parser(self):
+            raise ImportError("docling is required")
+
+        def parse_file(self, file, errors="ignore"):
+            return "never"
+
+    with pytest.raises(ImportError, match="docling is required"):
+        delegate_parse(_NeedsLib(), Path("x.pdf"), "ignore")

tests/parser/file/test_tableize.py

@@ -72,3 +72,23 @@ def test_non_table_text_passes_through_verbatim():
     """Only the trailing newline may differ (splitlines/join round-trip)."""
     md = "# Heading\n\nA paragraph with no numbers.\n\n- a list item\n"
     assert tableize(md) == md.rstrip("\n")
+
+
+def test_single_trailing_number_lines_are_not_a_table():
+    """Headings, footnotes and version lists look like 'word number' rows; leave them alone."""
+    from application.parser.file.tableize import tableize
+
+    for block in (
+        "Chapter 1\nChapter 2\nChapter 3",
+        "Footnote 1\nFootnote 2\nFootnote 3",
+        "ISO 9001\nISO 14001\nISO 27001",
+        "Version 2.0\nVersion 3.0\nVersion 4.0",
+    ):
+        assert tableize(block) == block
+
+
+def test_leader_rows_with_one_value_still_convert():
+    from application.parser.file.tableize import tableize
+
+    block = "Revenue ...... 1,234\nCosts ...... 567\nProfit ...... 667"
+    assert "| --- |" in tableize(block)

tests/parser/remote/test_s3_loader.py

@@ -903,3 +903,14 @@ class TestSSRFValidation:
         with pytest.raises(ValueError, match="Invalid S3 endpoint_url"):
             s3_loader.load_data(input_data)
         mock_boto3.client.assert_not_called()
+
+
+def test_is_supported_document_follows_the_upload_whitelist():
+    from application.parser.file.constants import SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS
+    from application.parser.remote.s3_loader import S3Loader
+
+    for suffix in SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS:
+        key = f"bucket/file{suffix}"
+        assert S3Loader.is_text_file(None, key) or S3Loader.is_supported_document(None, key), suffix
+    assert S3Loader.is_supported_document(None, "bucket/file.doc")
+    assert not S3Loader.is_supported_document(None, "bucket/file.exe")

tests/parser/test_document_reader.py

@@ -707,3 +707,49 @@ def test_structured_output_stays_docling_under_anydoc(monkeypatch):
     out = parse_document_bytes(b"%PDF-1.4", "doc.pdf", output="structured")
     assert out["output"] == "structured"
     assert len(calls) == 1
+
+
+# --- second-pass routing fixes ---------------------------------------------------
+
+
+def test_fast_engine_covers_the_whole_legacy_map():
+    """`fast` must never fall through to the configured (anydoc/docling) map."""
+    for suffix, name in {
+        ".xhtml": "HTMLParser",
+        ".pptx": "PPTXParser",
+        ".epub": "EpubParser",
+        ".rst": "RstParser",
+        ".json": "JSONParser",
+    }.items():
+        assert type(dr._legacy_parser_for(suffix)).__name__ == name
+
+
+def test_tables_are_not_collected_when_the_pdf_parser_is_not_docling(monkeypatch):
+    """Under OCR_BACKEND=native the docling engine hands PDFs to the native OCR
+    parser; a vanilla DocumentConverter table pass would OCR the scan again."""
+
+    from application.parser.file.ocr_parser import NativeOcrPdfParser
+
+    native = NativeOcrPdfParser()
+    native._parser_config = {}
+    monkeypatch.setattr(native, "parse_file", lambda path, errors="ignore": "native text")
+
+    calls = []
+    monkeypatch.setattr(dr, "_pick_parser", lambda *a, **k: native)
+    monkeypatch.setattr(dr, "_effective_engine", lambda engine: "docling")
+    monkeypatch.setattr(dr, "_docling_structured", lambda *a, **k: calls.append(1) or {"tables": []})
+    monkeypatch.setattr(dr, "_zip_bomb_reason", lambda *a, **k: None)
+
+    out = parse_document_bytes(b"%PDF-1.4", "scan.pdf", engine="docling", include_tables=True)
+
+    assert out["content"] == "native text"
+    assert calls == []
+
+
+def test_binary_office_suffix_without_anydoc_is_an_error_not_text(monkeypatch):
+    monkeypatch.setattr(dr, "_pick_parser", lambda *a, **k: None)
+    monkeypatch.setattr(dr, "_zip_bomb_reason", lambda *a, **k: None)
+
+    out = parse_document_bytes(b"\xd0\xcf\x11\xe0 OLE bytes", "legacy.doc", output="text")
+
+    assert "firecrawl-anydoc" in out["error"]