fix: handle non text uploads more carefully

This commit is contained in:
Alex committed 2026-09-14 17:27:13 +01:00
1 parent 82691314ca
commit 3030aba39d
21 files changed
+729 -40

No files matched your search

+8 -2
View File
@@ -527,22 +527,28 @@ def test_tableize_never_touches_docling_reroute(monkeypatch):
def test_scanned_pdf_with_near_empty_fallback_fails_loudly_when_ocr_off(tmp_path):
"""A scan whose fallback (OCR off) extracts almost nothing must fail with
an actionable message, not be stored as an empty document."""
from docsgpt.parser.file.base_parser import NoTextLayerError
path = _scanned_pdf(tmp_path / "scan.pdf")
fallback = _RecordingFallback(result=" ")
fallback.ocr_enabled = False
parser = AnydocParser(fallback_parser=fallback)
with pytest.raises(DocumentParseError, match="OCR_ENABLED"):
# Still a DocumentParseError for source ingestion, but typed, so the
# attachment worker can keep the file for models that read it natively.
with pytest.raises(NoTextLayerError, match="OCR_ENABLED"):
parser.parse_file(path)
assert parser.last_engine is None
def test_scanned_pdf_with_near_empty_fallback_and_ocr_on_reports_it(tmp_path):
from docsgpt.parser.file.base_parser import NoTextLayerError
path = _scanned_pdf(tmp_path / "scan.pdf")
fallback = _RecordingFallback(result="x")
fallback.ocr_enabled = True
with pytest.raises(DocumentParseError, match="even with OCR enabled"):
with pytest.raises(NoTextLayerError, match="even with OCR enabled"):
AnydocParser(fallback_parser=fallback).parse_file(path)
+40
View File
@@ -222,6 +222,31 @@ class TestSimpleDirectoryReaderLoadData:
with pytest.raises(DocumentParseError, match="InvalidCxxCompiler"):
reader.load_data()
def test_single_unparseable_file_keeps_the_parsers_error_type(self, tmp_path):
"""The attachment worker tells a scan from a broken file by the error's type.
Re-wrapping the parser's exception in a plain ``DocumentParseError``
would erase ``NoTextLayerError`` and turn every scanned PDF back into
a failed upload.
"""
from docsgpt.parser.file.base_parser import NoTextLayerError
from docsgpt.parser.file.bulk import SimpleDirectoryReader
(tmp_path / "scan.pdf").write_bytes(b"%PDF-1.7")
mock_parser = MagicMock()
mock_parser.parser_config_set = True
mock_parser.parse_file.side_effect = NoTextLayerError(
"scan.pdf appears to be a scanned PDF (no text layer)"
)
reader = SimpleDirectoryReader(
input_files=[str(tmp_path / "scan.pdf")],
file_extractor={".pdf": mock_parser},
)
with pytest.raises(NoTextLayerError, match="scanned PDF"):
reader.load_data()
def test_all_files_unparseable_raises(self, tmp_path):
"""Nothing parsed is a failed ingest, not an empty success."""
from docsgpt.parser.file.base_parser import DocumentParseError
@@ -588,6 +613,21 @@ class TestParserEngineSwitch:
assert type(extractor[".xlsx"].fallback_parser).__name__ == "ExcelParser"
assert type(extractor[".png"]).__name__ == "ImageParser"
def test_anydoc_without_docling_admits_every_image_format(self, monkeypatch):
"""The production image ships neither docling nor OCR. An image is
sent to the model as an image, so .webp/.tiff/.bmp need a parser
entry or the worker refuses them before any model sees them."""
pytest.importorskip("anydoc")
import sys
from docsgpt.parser.file.bulk import get_default_file_extractor
monkeypatch.setitem(sys.modules, "docling", None)
extractor = get_default_file_extractor(engine="anydoc")
for suffix in (".webp", ".tiff", ".tif", ".bmp"):
assert type(extractor[suffix]).__name__ == "ImageParser", suffix
def test_docling_engine_keeps_docling_map(self):
pytest.importorskip("docling")
from docsgpt.parser.file.bulk import get_default_file_extractor
@@ -0,0 +1,64 @@
"""Re-encoding images the model providers reject (TIFF, BMP) as PNG.
Chat attachments reach vision models as the stored file itself. OpenAI and
Anthropic accept png/jpeg/webp/gif only, so a TIFF fax or a BMP screenshot has
to be re-encoded before it can be sent at all.
"""
import io
import pytest
from PIL import Image
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.image_parser import (
VISION_CONVERTIBLE_MIME_TYPES,
convert_image_to_png,
)
def _encoded(fmt, frames=1, size=(40, 24)):
"""An in-memory image of ``frames`` pages, each a different red level."""
images = [Image.new("RGB", size, color=(i * 60, 20, 200)) for i in range(frames)]
buf = io.BytesIO()
if frames > 1:
images[0].save(buf, format=fmt, save_all=True, append_images=images[1:])
else:
images[0].save(buf, format=fmt)
buf.seek(0)
return buf
@pytest.mark.unit
class TestConvertImageToPng:
def test_tiff_becomes_png(self):
png, frames = convert_image_to_png(_encoded("TIFF"))
decoded = Image.open(io.BytesIO(png))
assert decoded.format == "PNG"
assert decoded.size == (40, 24)
assert frames == 1
def test_multi_page_tiff_keeps_the_first_page_and_reports_the_count(self):
png, frames = convert_image_to_png(_encoded("TIFF", frames=3))
decoded = Image.open(io.BytesIO(png)).convert("RGB")
assert frames == 3
assert decoded.getpixel((0, 0))[0] == 0 # page one's red level
def test_bmp_becomes_png(self):
png, frames = convert_image_to_png(_encoded("BMP"))
assert Image.open(io.BytesIO(png)).format == "PNG"
assert frames == 1
def test_unreadable_bytes_raise_a_parse_error(self):
with pytest.raises(DocumentParseError):
convert_image_to_png(io.BytesIO(b"not an image"))
def test_only_formats_the_providers_reject_are_converted(self):
assert {"image/tiff", "image/bmp"} <= VISION_CONVERTIBLE_MIME_TYPES
assert not (
{"image/png", "image/jpeg", "image/webp", "image/gif"}
& VISION_CONVERTIBLE_MIME_TYPES
)
+6 -3
View File
@@ -536,13 +536,16 @@ class TestNativeOcrImageParser:
@pytest.mark.unit
class TestExtractorWiring:
def test_legacy_map_without_ocr_is_unchanged(self):
def test_legacy_map_without_ocr_hands_every_image_to_image_parser(self):
"""Without OCR an image still reaches the model as an image, so every
image suffix the upload gate admits needs an entry. Only png/jpg/jpeg
had one, and a .webp was refused outright (prod 2026-09-10)."""
from docsgpt.parser.file.bulk import _legacy_file_extractor
extractor = _legacy_file_extractor()
assert type(extractor[".pdf"]).__name__ == "PDFParser"
assert type(extractor[".png"]).__name__ == "ImageParser"
assert ".tiff" not in extractor
for suffix in op.IMAGE_SUFFIXES:
assert type(extractor[suffix]).__name__ == "ImageParser", suffix
def test_legacy_map_with_ocr_uses_native_parsers(self):
from docsgpt.parser.file.bulk import _legacy_file_extractor