fix(attachments,usage): validate content behind a BOM, judge by the live parser table

Review follow-ups.

A BOM told the sniff which encoding to read, but was also taken as the
verdict: three prepended bytes let any binary through, including as
notes.txt. A BOM now only selects the test — UTF-8 falls through to the
byte rules on the remainder, UTF-16/32 decode and judge the characters
(NUL, unprintable, or replacement chars from bytes the decoder could not
read). Real Notepad-Unicode text still passes, mp4-behind-a-BOM does not,
in either language.

The gate treated the full parser table as a given, but without docling the
fallback extractor has no .tif/.tiff/.bmp/.webp/.vtt/.xml handler, so those
suffixes skipped the content check and reached the plain-text fallthrough —
the original bug, one install away. The worker now passes the keys of the
extractor it actually built, making the second gate stricter than the
route's static one rather than a copy of it.

Cache bins: extraction coerced a missing value to 0 and only non-zero bins
were recorded, so a provider reporting cached_tokens=0 persisted as NULL —
indistinguishable from "not reported", and OpenAI reports exactly that on
every uncached request. Bins are now carried as Optional and recorded when
not None, which is what the nullable columns and the NULL-means-unknown
comment already assumed. Anthropic's cache_read/cache_creation bins had the
same shape and are fixed alongside; the int-or-None coercion is shared in
llm/base.py.

application/llm/anthropic.py

@@ -6,7 +6,7 @@ from typing import Any, Dict, Generator, List, Optional, Tuple
 from anthropic import Anthropic
 
 from application.core.settings import settings
-from application.llm.base import BaseLLM
+from application.llm.base import BaseLLM, optional_int
 from application.storage.storage_creator import StorageCreator
 
 logger = logging.getLogger(__name__)
@@ -387,8 +387,12 @@ class AnthropicLLM(BaseLLM):
             # message_start are reused below instead.
             if not output_only:
                 base_input = int(getattr(usage, "input_tokens", 0) or 0)
-                created = int(getattr(usage, "cache_creation_input_tokens", 0) or 0)
-                cached = int(getattr(usage, "cache_read_input_tokens", 0) or 0)
+                # None (unreported) and 0 (reported, no caching) must stay
+                # distinguishable — see ``optional_int``.
+                created = optional_int(
+                    getattr(usage, "cache_creation_input_tokens", None)
+                )
+                cached = optional_int(getattr(usage, "cache_read_input_tokens", None))
         except (TypeError, ValueError):
             return
 
@@ -398,11 +402,11 @@ class AnthropicLLM(BaseLLM):
             prompt = int(previous.get("prompt_tokens", 0) or 0)
             details = previous.get("prompt_tokens_details")
         else:
-            prompt = base_input + created + cached
+            prompt = base_input + (created or 0) + (cached or 0)
             details = {}
-            if cached:
+            if cached is not None:
                 details["cached_tokens"] = cached
-            if created:
+            if created is not None:
                 details["cache_creation_tokens"] = created
             details = details or None
 

application/llm/base.py

@@ -33,6 +33,29 @@ _STREAM_RETRYABLE_TRANSPORT_ERRORS = (
 )
 
 
+def optional_int(value) -> Optional[int]:
+    """Coerce a provider-reported count to int, keeping "unreported" as None.
+
+    Cache bins persist as NULL when a provider says nothing and as 0 when it
+    reports zero, so the two must stay distinguishable all the way from the
+    usage object: providers report ``cached_tokens: 0`` on every uncached
+    request, and folding that into "unknown" would file the ordinary case as
+    no-data.
+
+    Args:
+        value: The raw attribute off a provider usage object.
+
+    Returns:
+        The integer value, or None when absent or not a number.
+    """
+    if value is None:
+        return None
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return None
+
+
 class BaseLLM(ABC):
     # Stamped onto the ``llm_stream_start`` event so dashboards can group
     # calls by vendor. Subclasses override.

application/llm/openai.py

@@ -9,7 +9,7 @@ from typing import Any, Callable
 from openai import BadRequestError, OpenAI
 
 from application.core.settings import settings
-from application.llm.base import BaseLLM
+from application.llm.base import BaseLLM, optional_int
 from application.storage.storage_creator import StorageCreator
 
 # Placeholder sent to OpenAI-compatible backends that require no credentials.
@@ -1305,14 +1305,11 @@ class OpenAILLM(BaseLLM):
         }
         input_details = getattr(usage, "prompt_tokens_details", None)
         output_details = getattr(usage, "completion_tokens_details", None)
-        try:
-            cached = int(getattr(input_details, "cached_tokens", 0) or 0)
-            written = int(getattr(input_details, "cache_write_tokens", 0) or 0)
-            reasoning = int(getattr(output_details, "reasoning_tokens", 0) or 0)
-        except (TypeError, ValueError):
-            cached = 0
-            written = 0
-            reasoning = 0
+        cached = optional_int(getattr(input_details, "cached_tokens", None))
+        written = optional_int(getattr(input_details, "cache_write_tokens", None))
+        reasoning = optional_int(
+            getattr(output_details, "reasoning_tokens", None)
+        )
         details = self._prompt_cache_details(cached, written)
         if details:
             result["prompt_tokens_details"] = details
@@ -1322,18 +1319,21 @@ class OpenAILLM(BaseLLM):
         self._last_usage_claimed = False
 
     @staticmethod
-    def _prompt_cache_details(cached: int, written: int) -> dict:
+    def _prompt_cache_details(cached: int | None, written: int | None) -> dict:
         """Build the ``prompt_tokens_details`` breakdown from the two cache bins.
 
-        Only non-zero bins are included so a provider that never reports
-        caching produces no details (and no misleading zero downstream).
-        ``cache_write_tokens`` is reported by newer OpenAI-family deployments
-        that charge for cache writes.
+        A bin is included whenever the provider reported it, zero included:
+        downstream, an absent bin persists as NULL ("we don't know") and a
+        reported zero as 0 ("no cache hits"), and OpenAI reports
+        ``cached_tokens: 0`` on every uncached request. Collapsing the two
+        would file every ordinary request as unknown. ``cache_write_tokens``
+        is reported by newer OpenAI-family deployments that charge for cache
+        writes.
         """
         details = {}
-        if cached:
+        if cached is not None:
             details["cached_tokens"] = cached
-        if written:
+        if written is not None:
             details["cache_write_tokens"] = written
         return details
 
@@ -1375,9 +1375,13 @@ class OpenAILLM(BaseLLM):
                 "completion_tokens": completion,
                 "total_tokens": int(getattr(usage, "total_tokens", 0) or prompt + completion),
             }
-            cached = int(getattr(input_details, "cached_tokens", 0) or 0)
-            written = int(getattr(input_details, "cache_write_tokens", 0) or 0)
-            reasoning = int(getattr(output_details, "reasoning_tokens", 0) or 0)
+            cached = optional_int(getattr(input_details, "cached_tokens", None))
+            written = optional_int(
+                getattr(input_details, "cache_write_tokens", None)
+            )
+            reasoning = optional_int(
+                getattr(output_details, "reasoning_tokens", None)
+            )
             details = self._prompt_cache_details(cached, written)
             if details:
                 result["prompt_tokens_details"] = details

application/upload_limits.py

@@ -2,10 +2,11 @@
 
 from __future__ import annotations
 
+import codecs
 import io
 import os
 from contextlib import suppress
-from typing import BinaryIO
+from typing import BinaryIO, Container
 
 from application.core.settings import settings
 from application.parser.file.constants import (
@@ -64,16 +65,48 @@ _TEXT_SNIFF_BYTES = 8192
 _MAX_NONTEXT_RATIO = 0.10
 # Control bytes that occur in ordinary text: tab, LF, VT, FF, CR, ESC.
 _TEXT_CONTROL_BYTES = frozenset({0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x1B})
-# A UTF-16/32 file is half NUL bytes, so it has to be recognised by its BOM
-# before the NUL test — Notepad's "Unicode" .txt is ordinary text.
+_TEXT_CONTROL_CHARS = frozenset("\t\n\v\f\r\x1b")
+# A UTF-16/32 file is half NUL bytes, so the byte rules below would reject it.
+# Its BOM says which encoding to read it as; the decoded characters are then
+# judged instead. Longest BOM first — UTF-32-LE starts with the UTF-16-LE one.
+# A BOM is never a verdict on its own: three bytes must not buy a video a pass.
 _TEXT_BOMS = (
-    b"\xef\xbb\xbf",
-    b"\xff\xfe",
-    b"\xfe\xff",
-    b"\x00\x00\xfe\xff",
+    (codecs.BOM_UTF32_LE, "utf-32-le"),
+    (codecs.BOM_UTF32_BE, "utf-32-be"),
+    (codecs.BOM_UTF8, None),
+    (codecs.BOM_UTF16_LE, "utf-16-le"),
+    (codecs.BOM_UTF16_BE, "utf-16-be"),
 )
 
 
+def _decoded_looks_like_text(text: str) -> bool:
+    """Return whether decoded characters read as text.
+
+    Args:
+        text: Characters decoded from a BOM-marked sample.
+
+    Returns:
+        False when a NUL character appears — the byte-level rule, one level
+        up, since no text holds one — or when too many characters are
+        unprintable: control, unassigned, private-use or surrogate, which is
+        what binary decodes into. Replacement characters count as unprintable,
+        so bytes the decoder could not read are evidence rather than silently
+        dropped (a truncated character at the sample boundary is one of
+        thousands and cannot swing the ratio).
+    """
+    if not text:
+        return True
+    if "\x00" in text:
+        return False
+    nontext = sum(
+        1
+        for char in text
+        if char == "�"
+        or (not char.isprintable() and char not in _TEXT_CONTROL_CHARS)
+    )
+    return nontext / len(text) <= _MAX_NONTEXT_RATIO
+
+
 def looks_like_text(sample: bytes) -> bool:
     """Return whether a leading byte sample reads as text rather than binary.
 
@@ -82,20 +115,30 @@ def looks_like_text(sample: bytes) -> bool:
 
     Returns:
         False when the sample holds a NUL byte or too many other non-text
-        control bytes, True otherwise. A Unicode BOM settles it as text.
+        control bytes, True otherwise. A UTF-16/32 BOM switches the test to
+        the decoded characters; the content is still what decides.
     """
     if not sample:
         return True
-    if sample.startswith(_TEXT_BOMS):
+    body = sample
+    for bom, encoding in _TEXT_BOMS:
+        if not sample.startswith(bom):
+            continue
+        body = sample[len(bom) :]
+        if encoding is not None:
+            return _decoded_looks_like_text(body.decode(encoding, errors="replace"))
+        # UTF-8 BOM: the byte rules still apply to everything after it.
+        break
+    if not body:
         return True
-    if b"\x00" in sample:
+    if b"\x00" in body:
         return False
     nontext = sum(
         1
-        for byte in sample
+        for byte in body
         if (byte < 0x20 and byte not in _TEXT_CONTROL_BYTES) or byte == 0x7F
     )
-    return nontext / len(sample) <= _MAX_NONTEXT_RATIO
+    return nontext / len(body) <= _MAX_NONTEXT_RATIO
 
 
 def file_looks_like_text(path: str | os.PathLike[str]) -> bool:
@@ -116,25 +159,35 @@ def file_looks_like_text(path: str | os.PathLike[str]) -> bool:
 
 
 def enforce_parseable_attachment(
-    path: str | os.PathLike[str], filename: str | None
+    path: str | os.PathLike[str],
+    filename: str | None,
+    parser_extensions: Container[str] | None = None,
 ) -> None:
     """Reject an attachment that no parser handles and that is not plain text.
 
-    Suffixes with a parser (``ATTACHMENT_PARSER_EXTENSIONS``) are admitted
-    unconditionally — a PDF is binary and parses fine. Everything else, .txt
-    included, has to read as text: that is what keeps a video or an archive
-    out of the plain-text fallthrough while leaving source, config and log
-    files in, whatever the file happens to be named.
+    Suffixes with a parser are admitted unconditionally — a PDF is binary and
+    parses fine. Everything else, .txt included, has to read as text: that is
+    what keeps a video or an archive out of the plain-text fallthrough while
+    leaving source, config and log files in, whatever the file is named.
 
     Args:
         path: Local path to the staged upload, readable before it is stored.
         filename: The upload's filename, used for the suffix and the message.
+        parser_extensions: Suffixes to treat as parser-backed. Defaults to
+            ``ATTACHMENT_PARSER_EXTENSIONS``, which assumes the full parser
+            table; callers holding the live extractor (the worker) pass its
+            keys, so a docling-less install does not admit a .webp on trust
+            and then read it as text.
 
     Raises:
         UnsupportedUploadTypeError: When the file has no parser and its
             contents are binary.
     """
-    if has_attachment_parser(filename):
+    if parser_extensions is None:
+        has_parser = has_attachment_parser(filename)
+    else:
+        has_parser = attachment_extension(filename) in parser_extensions
+    if has_parser:
         return
     if file_looks_like_text(path):
         return

application/worker.py

@@ -1558,22 +1558,26 @@ class AttachmentRejectedError(Exception):
     """
 
 
-def _reject_unparseable_attachment(local_path: str, filename: str) -> None:
+def _reject_unparseable_attachment(
+    local_path: str, filename: str, parser_extensions
+) -> None:
     """Reject an attachment with no parser whose contents are binary.
 
-    Defence in depth behind the route's gate: ``SimpleDirectoryReader`` opens
-    any suffix it has no parser for as plain text, so a binary must fail here,
-    visibly, rather than "succeed" as garbage.
+    Defence in depth behind the route's gate, and stricter than it: this runs
+    against the parser table actually loaded, so a suffix the route trusted
+    (.webp with docling absent, say) is still refused rather than opened as
+    plain text by ``SimpleDirectoryReader`` and stored as garbage.
 
     Args:
         local_path: Filesystem path of the attachment about to be parsed.
         filename: The upload's original filename, which carries the suffix.
+        parser_extensions: Suffixes the live ``file_extractor`` can parse.
 
     Raises:
         AttachmentRejectedError: If the file has no parser and is not text.
     """
     try:
-        enforce_parseable_attachment(local_path, filename)
+        enforce_parseable_attachment(local_path, filename, parser_extensions)
     except UnsupportedUploadTypeError as exc:
         raise AttachmentRejectedError(str(exc)) from exc
 
@@ -1775,7 +1779,7 @@ def attachment_worker(self, file_info, user):
         parser_name = type(_parser).__name__ if _parser is not None else "SimpleDirectoryReader"
 
         def _parse_local_file(local_path: str, **kwargs) -> Document:
-            _reject_unparseable_attachment(local_path, filename)
+            _reject_unparseable_attachment(local_path, filename, set(file_extractor))
             _reject_attachment_zip_bomb(local_path)
             parse_path, is_temp_copy = _bounded_attachment_copy(local_path)
             try: