diff --git a/application/llm/anthropic.py b/application/llm/anthropic.py index 7944dff6..f26a00ac 100644 --- a/application/llm/anthropic.py +++ b/application/llm/anthropic.py @@ -6,7 +6,7 @@ from typing import Any, Dict, Generator, List, Optional, Tuple from anthropic import Anthropic from application.core.settings import settings -from application.llm.base import BaseLLM +from application.llm.base import BaseLLM, optional_int from application.storage.storage_creator import StorageCreator logger = logging.getLogger(__name__) @@ -387,8 +387,12 @@ class AnthropicLLM(BaseLLM): # message_start are reused below instead. if not output_only: base_input = int(getattr(usage, "input_tokens", 0) or 0) - created = int(getattr(usage, "cache_creation_input_tokens", 0) or 0) - cached = int(getattr(usage, "cache_read_input_tokens", 0) or 0) + # None (unreported) and 0 (reported, no caching) must stay + # distinguishable — see ``optional_int``. + created = optional_int( + getattr(usage, "cache_creation_input_tokens", None) + ) + cached = optional_int(getattr(usage, "cache_read_input_tokens", None)) except (TypeError, ValueError): return @@ -398,11 +402,11 @@ class AnthropicLLM(BaseLLM): prompt = int(previous.get("prompt_tokens", 0) or 0) details = previous.get("prompt_tokens_details") else: - prompt = base_input + created + cached + prompt = base_input + (created or 0) + (cached or 0) details = {} - if cached: + if cached is not None: details["cached_tokens"] = cached - if created: + if created is not None: details["cache_creation_tokens"] = created details = details or None diff --git a/application/llm/base.py b/application/llm/base.py index d331ca17..2afa2f2a 100644 --- a/application/llm/base.py +++ b/application/llm/base.py @@ -33,6 +33,29 @@ _STREAM_RETRYABLE_TRANSPORT_ERRORS = ( ) +def optional_int(value) -> Optional[int]: + """Coerce a provider-reported count to int, keeping "unreported" as None. + + Cache bins persist as NULL when a provider says nothing and as 0 when it + reports zero, so the two must stay distinguishable all the way from the + usage object: providers report ``cached_tokens: 0`` on every uncached + request, and folding that into "unknown" would file the ordinary case as + no-data. + + Args: + value: The raw attribute off a provider usage object. + + Returns: + The integer value, or None when absent or not a number. + """ + if value is None: + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + class BaseLLM(ABC): # Stamped onto the ``llm_stream_start`` event so dashboards can group # calls by vendor. Subclasses override. diff --git a/application/llm/openai.py b/application/llm/openai.py index 65751206..6fcc8bc3 100644 --- a/application/llm/openai.py +++ b/application/llm/openai.py @@ -9,7 +9,7 @@ from typing import Any, Callable from openai import BadRequestError, OpenAI from application.core.settings import settings -from application.llm.base import BaseLLM +from application.llm.base import BaseLLM, optional_int from application.storage.storage_creator import StorageCreator # Placeholder sent to OpenAI-compatible backends that require no credentials. @@ -1305,14 +1305,11 @@ class OpenAILLM(BaseLLM): } input_details = getattr(usage, "prompt_tokens_details", None) output_details = getattr(usage, "completion_tokens_details", None) - try: - cached = int(getattr(input_details, "cached_tokens", 0) or 0) - written = int(getattr(input_details, "cache_write_tokens", 0) or 0) - reasoning = int(getattr(output_details, "reasoning_tokens", 0) or 0) - except (TypeError, ValueError): - cached = 0 - written = 0 - reasoning = 0 + cached = optional_int(getattr(input_details, "cached_tokens", None)) + written = optional_int(getattr(input_details, "cache_write_tokens", None)) + reasoning = optional_int( + getattr(output_details, "reasoning_tokens", None) + ) details = self._prompt_cache_details(cached, written) if details: result["prompt_tokens_details"] = details @@ -1322,18 +1319,21 @@ class OpenAILLM(BaseLLM): self._last_usage_claimed = False @staticmethod - def _prompt_cache_details(cached: int, written: int) -> dict: + def _prompt_cache_details(cached: int | None, written: int | None) -> dict: """Build the ``prompt_tokens_details`` breakdown from the two cache bins. - Only non-zero bins are included so a provider that never reports - caching produces no details (and no misleading zero downstream). - ``cache_write_tokens`` is reported by newer OpenAI-family deployments - that charge for cache writes. + A bin is included whenever the provider reported it, zero included: + downstream, an absent bin persists as NULL ("we don't know") and a + reported zero as 0 ("no cache hits"), and OpenAI reports + ``cached_tokens: 0`` on every uncached request. Collapsing the two + would file every ordinary request as unknown. ``cache_write_tokens`` + is reported by newer OpenAI-family deployments that charge for cache + writes. """ details = {} - if cached: + if cached is not None: details["cached_tokens"] = cached - if written: + if written is not None: details["cache_write_tokens"] = written return details @@ -1375,9 +1375,13 @@ class OpenAILLM(BaseLLM): "completion_tokens": completion, "total_tokens": int(getattr(usage, "total_tokens", 0) or prompt + completion), } - cached = int(getattr(input_details, "cached_tokens", 0) or 0) - written = int(getattr(input_details, "cache_write_tokens", 0) or 0) - reasoning = int(getattr(output_details, "reasoning_tokens", 0) or 0) + cached = optional_int(getattr(input_details, "cached_tokens", None)) + written = optional_int( + getattr(input_details, "cache_write_tokens", None) + ) + reasoning = optional_int( + getattr(output_details, "reasoning_tokens", None) + ) details = self._prompt_cache_details(cached, written) if details: result["prompt_tokens_details"] = details diff --git a/application/upload_limits.py b/application/upload_limits.py index 3c24c2de..7ea3b0a8 100644 --- a/application/upload_limits.py +++ b/application/upload_limits.py @@ -2,10 +2,11 @@ from __future__ import annotations +import codecs import io import os from contextlib import suppress -from typing import BinaryIO +from typing import BinaryIO, Container from application.core.settings import settings from application.parser.file.constants import ( @@ -64,16 +65,48 @@ _TEXT_SNIFF_BYTES = 8192 _MAX_NONTEXT_RATIO = 0.10 # Control bytes that occur in ordinary text: tab, LF, VT, FF, CR, ESC. _TEXT_CONTROL_BYTES = frozenset({0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x1B}) -# A UTF-16/32 file is half NUL bytes, so it has to be recognised by its BOM -# before the NUL test — Notepad's "Unicode" .txt is ordinary text. +_TEXT_CONTROL_CHARS = frozenset("\t\n\v\f\r\x1b") +# A UTF-16/32 file is half NUL bytes, so the byte rules below would reject it. +# Its BOM says which encoding to read it as; the decoded characters are then +# judged instead. Longest BOM first — UTF-32-LE starts with the UTF-16-LE one. +# A BOM is never a verdict on its own: three bytes must not buy a video a pass. _TEXT_BOMS = ( - b"\xef\xbb\xbf", - b"\xff\xfe", - b"\xfe\xff", - b"\x00\x00\xfe\xff", + (codecs.BOM_UTF32_LE, "utf-32-le"), + (codecs.BOM_UTF32_BE, "utf-32-be"), + (codecs.BOM_UTF8, None), + (codecs.BOM_UTF16_LE, "utf-16-le"), + (codecs.BOM_UTF16_BE, "utf-16-be"), ) +def _decoded_looks_like_text(text: str) -> bool: + """Return whether decoded characters read as text. + + Args: + text: Characters decoded from a BOM-marked sample. + + Returns: + False when a NUL character appears — the byte-level rule, one level + up, since no text holds one — or when too many characters are + unprintable: control, unassigned, private-use or surrogate, which is + what binary decodes into. Replacement characters count as unprintable, + so bytes the decoder could not read are evidence rather than silently + dropped (a truncated character at the sample boundary is one of + thousands and cannot swing the ratio). + """ + if not text: + return True + if "\x00" in text: + return False + nontext = sum( + 1 + for char in text + if char == "�" + or (not char.isprintable() and char not in _TEXT_CONTROL_CHARS) + ) + return nontext / len(text) <= _MAX_NONTEXT_RATIO + + def looks_like_text(sample: bytes) -> bool: """Return whether a leading byte sample reads as text rather than binary. @@ -82,20 +115,30 @@ def looks_like_text(sample: bytes) -> bool: Returns: False when the sample holds a NUL byte or too many other non-text - control bytes, True otherwise. A Unicode BOM settles it as text. + control bytes, True otherwise. A UTF-16/32 BOM switches the test to + the decoded characters; the content is still what decides. """ if not sample: return True - if sample.startswith(_TEXT_BOMS): + body = sample + for bom, encoding in _TEXT_BOMS: + if not sample.startswith(bom): + continue + body = sample[len(bom) :] + if encoding is not None: + return _decoded_looks_like_text(body.decode(encoding, errors="replace")) + # UTF-8 BOM: the byte rules still apply to everything after it. + break + if not body: return True - if b"\x00" in sample: + if b"\x00" in body: return False nontext = sum( 1 - for byte in sample + for byte in body if (byte < 0x20 and byte not in _TEXT_CONTROL_BYTES) or byte == 0x7F ) - return nontext / len(sample) <= _MAX_NONTEXT_RATIO + return nontext / len(body) <= _MAX_NONTEXT_RATIO def file_looks_like_text(path: str | os.PathLike[str]) -> bool: @@ -116,25 +159,35 @@ def file_looks_like_text(path: str | os.PathLike[str]) -> bool: def enforce_parseable_attachment( - path: str | os.PathLike[str], filename: str | None + path: str | os.PathLike[str], + filename: str | None, + parser_extensions: Container[str] | None = None, ) -> None: """Reject an attachment that no parser handles and that is not plain text. - Suffixes with a parser (``ATTACHMENT_PARSER_EXTENSIONS``) are admitted - unconditionally — a PDF is binary and parses fine. Everything else, .txt - included, has to read as text: that is what keeps a video or an archive - out of the plain-text fallthrough while leaving source, config and log - files in, whatever the file happens to be named. + Suffixes with a parser are admitted unconditionally — a PDF is binary and + parses fine. Everything else, .txt included, has to read as text: that is + what keeps a video or an archive out of the plain-text fallthrough while + leaving source, config and log files in, whatever the file is named. Args: path: Local path to the staged upload, readable before it is stored. filename: The upload's filename, used for the suffix and the message. + parser_extensions: Suffixes to treat as parser-backed. Defaults to + ``ATTACHMENT_PARSER_EXTENSIONS``, which assumes the full parser + table; callers holding the live extractor (the worker) pass its + keys, so a docling-less install does not admit a .webp on trust + and then read it as text. Raises: UnsupportedUploadTypeError: When the file has no parser and its contents are binary. """ - if has_attachment_parser(filename): + if parser_extensions is None: + has_parser = has_attachment_parser(filename) + else: + has_parser = attachment_extension(filename) in parser_extensions + if has_parser: return if file_looks_like_text(path): return diff --git a/application/worker.py b/application/worker.py index a1507239..168985c6 100755 --- a/application/worker.py +++ b/application/worker.py @@ -1558,22 +1558,26 @@ class AttachmentRejectedError(Exception): """ -def _reject_unparseable_attachment(local_path: str, filename: str) -> None: +def _reject_unparseable_attachment( + local_path: str, filename: str, parser_extensions +) -> None: """Reject an attachment with no parser whose contents are binary. - Defence in depth behind the route's gate: ``SimpleDirectoryReader`` opens - any suffix it has no parser for as plain text, so a binary must fail here, - visibly, rather than "succeed" as garbage. + Defence in depth behind the route's gate, and stricter than it: this runs + against the parser table actually loaded, so a suffix the route trusted + (.webp with docling absent, say) is still refused rather than opened as + plain text by ``SimpleDirectoryReader`` and stored as garbage. Args: local_path: Filesystem path of the attachment about to be parsed. filename: The upload's original filename, which carries the suffix. + parser_extensions: Suffixes the live ``file_extractor`` can parse. Raises: AttachmentRejectedError: If the file has no parser and is not text. """ try: - enforce_parseable_attachment(local_path, filename) + enforce_parseable_attachment(local_path, filename, parser_extensions) except UnsupportedUploadTypeError as exc: raise AttachmentRejectedError(str(exc)) from exc @@ -1775,7 +1779,7 @@ def attachment_worker(self, file_info, user): parser_name = type(_parser).__name__ if _parser is not None else "SimpleDirectoryReader" def _parse_local_file(local_path: str, **kwargs) -> Document: - _reject_unparseable_attachment(local_path, filename) + _reject_unparseable_attachment(local_path, filename, set(file_extractor)) _reject_attachment_zip_bomb(local_path) parse_path, is_temp_copy = _bounded_attachment_copy(local_path) try: diff --git a/frontend/src/constants/fileUpload.test.ts b/frontend/src/constants/fileUpload.test.ts index 02579e76..c3fa4839 100644 --- a/frontend/src/constants/fileUpload.test.ts +++ b/frontend/src/constants/fileUpload.test.ts @@ -95,13 +95,28 @@ describe('looksLikeText', () => { }); it('accepts BOM-marked Unicode, which is half NUL bytes but still text', () => { - // UTF-8, UTF-16 LE/BE, UTF-32 BE. + // UTF-8, UTF-16 LE/BE, and UTF-32 BE (no TextDecoder — left to the server). expect(looksLikeText(bytes(0xef, 0xbb, 0xbf, 0x68, 0x69))).toBe(true); expect(looksLikeText(bytes(0xff, 0xfe, 0x68, 0x00, 0x69, 0x00))).toBe(true); expect(looksLikeText(bytes(0xfe, 0xff, 0x00, 0x68, 0x00, 0x69))).toBe(true); expect(looksLikeText(bytes(0x00, 0x00, 0xfe, 0xff, 0x00, 0x68))).toBe(true); }); + it('still rejects binary behind a BOM — three bytes buy nothing', () => { + // UTF-8 BOM + MP4 header: the byte rules apply to what follows it. + expect(looksLikeText(bytes(0xef, 0xbb, 0xbf, ...MP4_HEADER))).toBe(false); + // UTF-16 LE BOM + control-only code points. + expect( + looksLikeText( + bytes(0xff, 0xfe, 0x01, 0x00, 0x02, 0x00, 0x03, 0x00, 0x04, 0x00), + ), + ).toBe(false); + // UTF-16 BE BOM + bytes that decode to NUL characters. + expect(looksLikeText(bytes(0xfe, 0xff, 0x00, 0x00, 0x00, 0x41))).toBe( + false, + ); + }); + it('rejects a NUL byte and dense control bytes', () => { expect(looksLikeText(MP4_HEADER)).toBe(false); expect(looksLikeText(bytes(0x74, 0x78, 0x00, 0x74))).toBe(false); @@ -142,9 +157,11 @@ describe('partitionAttachmentFiles', () => { it('refuses a binary renamed to .txt — .txt is sniffed like any other suffix', async () => { const { supported, unsupported } = await partitionAttachmentFiles([ file('notes.txt', MP4_HEADER), + // A BOM in front of it changes nothing. + file('bom.txt', bytes(0xef, 0xbb, 0xbf, ...MP4_HEADER)), ]); expect(supported).toEqual([]); - expect(unsupported.map((f) => f.name)).toEqual(['notes.txt']); + expect(unsupported.map((f) => f.name)).toEqual(['notes.txt', 'bom.txt']); }); it('admits parser-backed types on their suffix, binary contents and all', async () => { diff --git a/frontend/src/constants/fileUpload.ts b/frontend/src/constants/fileUpload.ts index d68b2aa3..ae5e60ba 100644 --- a/frontend/src/constants/fileUpload.ts +++ b/frontend/src/constants/fileUpload.ts @@ -160,27 +160,82 @@ const TEXT_SNIFF_BYTES = 8192; const MAX_NONTEXT_RATIO = 0.1; // Control bytes that occur in ordinary text: tab, LF, VT, FF, CR, ESC. const TEXT_CONTROL_BYTES = new Set([0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x1b]); -// A UTF-16/32 file is half NUL bytes, so it has to be recognised by its BOM -// before the NUL test — Notepad's "Unicode" .txt is ordinary text. -const TEXT_BOMS = [ - [0xef, 0xbb, 0xbf], - [0xff, 0xfe], - [0xfe, 0xff], - [0x00, 0x00, 0xfe, 0xff], +const TEXT_CONTROL_CHARS = new Set(['\t', '\n', '\v', '\f', '\r', '\u001b']); +// \p{C}: control, format, private-use, surrogate and unassigned code points — +// what binary decodes into. +const NONTEXT_CHAR = /\p{C}/u; +// A UTF-16/32 file is half NUL bytes, so the byte rules below would reject it. +// Its BOM says which encoding to read it as; the decoded characters are then +// judged instead. Longest BOM first — UTF-32-LE starts with the UTF-16-LE one. +// A BOM is never a verdict on its own: three bytes must not buy a video a pass. +const TEXT_BOMS: { bom: number[]; encoding: string | null }[] = [ + // TextDecoder has no UTF-32; leave those two to the server, which does. + { bom: [0xff, 0xfe, 0x00, 0x00], encoding: null }, + { bom: [0x00, 0x00, 0xfe, 0xff], encoding: null }, + { bom: [0xef, 0xbb, 0xbf], encoding: 'utf-8' }, + { bom: [0xff, 0xfe], encoding: 'utf-16le' }, + { bom: [0xfe, 0xff], encoding: 'utf-16be' }, ]; -/** Whether a leading byte sample reads as text rather than binary. */ +/** + * Whether decoded characters read as text rather than decoded binary. + * Replacement characters count against it, so bytes the decoder could not + * read are evidence rather than silently dropped. + */ +function decodedLooksLikeText(text: string): boolean { + // The byte-level NUL rule, one level up: no text holds a NUL character. + if (text.includes('\u0000')) return false; + let total = 0; + let nontext = 0; + for (const char of text) { + total += 1; + if ( + char === '\ufffd' || + (!TEXT_CONTROL_CHARS.has(char) && NONTEXT_CHAR.test(char)) + ) + nontext += 1; + } + if (total === 0) return true; + return nontext / total <= MAX_NONTEXT_RATIO; +} + +/** + * Whether a leading byte sample reads as text rather than binary. A UTF-16 + * BOM switches the test to the decoded characters; the content is still what + * decides. Mirrors `looks_like_text` in application/upload_limits.py. + */ export function looksLikeText(sample: Uint8Array): boolean { if (sample.length === 0) return true; - if (TEXT_BOMS.some((bom) => bom.every((byte, i) => sample[i] === byte))) - return true; + + let body = sample; + const marked = TEXT_BOMS.find(({ bom }) => + bom.every((byte, i) => sample[i] === byte), + ); + if (marked) { + // UTF-32, or a decoder this browser lacks: defer to the server rather + // than refuse a file it would accept. + if (marked.encoding === null) return true; + body = sample.subarray(marked.bom.length); + if (body.length === 0) return true; + if (marked.encoding !== 'utf-8') { + try { + return decodedLooksLikeText( + new TextDecoder(marked.encoding).decode(body), + ); + } catch { + return true; + } + } + // UTF-8 BOM: the byte rules still apply to everything after it. + } + let nontext = 0; - for (const byte of sample) { + for (const byte of body) { if (byte === 0x00) return false; if ((byte < 0x20 && !TEXT_CONTROL_BYTES.has(byte)) || byte === 0x7f) nontext += 1; } - return nontext / sample.length <= MAX_NONTEXT_RATIO; + return nontext / body.length <= MAX_NONTEXT_RATIO; } async function isSupportedAttachmentFile(file: File): Promise { diff --git a/tests/llm/test_openai_responses.py b/tests/llm/test_openai_responses.py index ab00a543..de72eb93 100644 --- a/tests/llm/test_openai_responses.py +++ b/tests/llm/test_openai_responses.py @@ -1583,5 +1583,26 @@ def test_record_responses_metadata_captures_cache_write_bin(monkeypatch): ), )) # A pure cache-write turn (first request of a prefix) has reads=0 and - # writes>0; the details must still be recorded. - assert llm._last_usage["prompt_tokens_details"] == {"cache_write_tokens": 9} + # writes>0; both bins were reported, so both are recorded — a reported + # zero is "no cache hits", not "unknown". + assert llm._last_usage["prompt_tokens_details"] == { + "cached_tokens": 0, + "cache_write_tokens": 9, + } + + +@pytest.mark.unit +def test_record_responses_metadata_omits_unreported_cache_bins(monkeypatch): + """A provider that says nothing about caching must persist as NULL, not 0.""" + llm = _make_llm(monkeypatch, _responses_caps()) + llm._record_responses_metadata(_ns( + id="resp_2", + usage=_ns( + input_tokens=10, + output_tokens=7, + total_tokens=17, + input_tokens_details=None, + output_tokens_details=None, + ), + )) + assert "prompt_tokens_details" not in llm._last_usage diff --git a/tests/test_upload_limits.py b/tests/test_upload_limits.py index 87c868f3..9d41e45a 100644 --- a/tests/test_upload_limits.py +++ b/tests/test_upload_limits.py @@ -121,6 +121,49 @@ def test_enforce_parseable_attachment_accepts_bom_marked_unicode_text( enforce_parseable_attachment(path, "notes.txt") +@pytest.mark.parametrize( + "bom", + [codecs.BOM_UTF8, codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE, codecs.BOM_UTF32_BE], +) +def test_enforce_parseable_attachment_rejects_binary_behind_a_bom(bom, tmp_path): + """A BOM says which encoding to read, not that the content is text. + + Otherwise three prepended bytes buy any binary a pass. + """ + from application.upload_limits import ( + enforce_parseable_attachment, + UnsupportedUploadTypeError, + ) + + path = tmp_path / "notes.txt" + path.write_bytes(bom + MP4_HEADER + bytes(range(256)) * 8) + + with pytest.raises(UnsupportedUploadTypeError): + enforce_parseable_attachment(path, "notes.txt") + + +def test_enforce_parseable_attachment_uses_the_extractor_it_is_given(tmp_path): + """The worker holds the live parser table; a trimmed install must not admit on trust. + + Without docling the fallback extractor has no .webp handler, so a .webp + would otherwise skip the content check and be read as plain text. + """ + from application.upload_limits import ( + enforce_parseable_attachment, + UnsupportedUploadTypeError, + ) + + path = tmp_path / "scan.webp" + path.write_bytes(b"RIFF\x00\x00\x00\x00WEBPVP8 " + bytes(range(256))) + + # Default list: .webp is parser-backed, admitted on its name. + enforce_parseable_attachment(path, "scan.webp") + + # The extractor actually loaded has no .webp parser. + with pytest.raises(UnsupportedUploadTypeError): + enforce_parseable_attachment(path, "scan.webp", {".pdf", ".docx"}) + + @pytest.mark.parametrize( "filename", [ @@ -177,6 +220,10 @@ def test_enforce_parseable_attachment_accepts_text_without_a_parser(filename, tm (b"\x1b[31mred log line\x1b[0m\n", True), (codecs.BOM_UTF16_LE + "hi\n".encode("utf-16-le"), True), (codecs.BOM_UTF8 + b"hi\n", True), + (codecs.BOM_UTF32_LE + "hi\n".encode("utf-32-le"), True), + # A BOM in front of binary is still binary. + (codecs.BOM_UTF8 + b"\x00\x01\x02", False), + (codecs.BOM_UTF16_LE + MP4_HEADER, False), (b"text\x00with nul", False), (bytes(range(32)) * 4, False), (b"\x7f\x7f\x7f\x7f" + b"a" * 16, False), diff --git a/tests/worker/test_attachment_worker.py b/tests/worker/test_attachment_worker.py index e5790cae..a48ef4ec 100644 --- a/tests/worker/test_attachment_worker.py +++ b/tests/worker/test_attachment_worker.py @@ -379,6 +379,53 @@ class TestAttachmentTypeGuard: failed = [payload for name, payload in events if name == "attachment.failed"] assert failed and failed[0]["error"] == "Unsupported file type: .mp4" + def test_suffix_the_loaded_extractor_cannot_parse_is_refused( + self, pg_conn, patch_worker_db, task_self, monkeypatch, tmp_path + ): + """The guard judges against the parser table actually loaded. + + Without docling the fallback extractor has no .webp parser, so a + .webp the route admitted on its name would otherwise be opened as + plain text here. + """ + from application import worker + + local_path = tmp_path / "scan.webp" + local_path.write_bytes(b"RIFF\x00\x00\x00\x00WEBPVP8 " + bytes(range(256))) + + events = [] + fake_storage = MagicMock(name="storage") + fake_storage.process_file.side_effect = lambda path, callback: callback( + str(local_path) + ) + monkeypatch.setattr(worker.StorageCreator, "get_storage", lambda: fake_storage) + # The docling-less fallback table: images are handled, .webp is not. + monkeypatch.setattr( + worker, + "get_default_file_extractor", + lambda ocr_enabled=False, pdf_text_fast_path=False: {".png": object()}, + ) + monkeypatch.setattr( + worker, + "publish_user_event", + lambda user, name, payload, **kwargs: events.append((name, payload)), + ) + + file_info = { + "filename": "scan.webp", + "attachment_id": "507f1f77bcf86cd799439014", + "path": "uploads/user1/attachments/scan.webp", + "metadata": {"source": "chat"}, + } + + with pytest.raises( + worker.AttachmentRejectedError, match=r"Unsupported file type: \.webp" + ): + worker.attachment_worker(task_self, file_info, "user1") + + failed = [payload for name, payload in events if name == "attachment.failed"] + assert failed and failed[0]["error"] == "Unsupported file type: .webp" + def test_text_without_a_parser_is_parsed( self, pg_conn, patch_worker_db, task_self, monkeypatch, tmp_path ):