fix(attachments): close the .txt gap in the parseability gate

Review follow-ups on the attachment gate.

.txt was listed as parser-backed, but it has no parser — it *is* the
plain-text fallthrough. That let it skip the content check, so renaming a
video to notes.txt walked straight back into the bug the gate exists for
(verified: 5132 chars of binary "extracted" and stored). The list is now
exactly the file extractor's keys, .txt included in the content check like
any other unparsed suffix, and the drift test asserts equality rather than
containment. The sniff now recognises a UTF-16/32 BOM as text, so a
Notepad "Unicode" .txt is not caught by the NUL-byte rule.

The picker's accept filter listed parser-backed suffixes only, hiding .txt,
.py and .log — files the gate reads happily — behind "All files". It now
carries text/* as well, so it can never be narrower than what the upload
accepts.

A rejected batch carries one errors entry per file, but the non-200 branch
applied the top-level message to every chip, so two files failing for two
reasons both reported the first one. Reasons are now matched by
upload_index, with the top-level message as fallback.

_get_store_attachment_user_error no longer reads str(exc): the
unsupported-type message is rebuilt from the filename, so no exception
state can reach a response body (CodeQL py/stack-trace-exposure).
This commit is contained in:
Alex committed 2026-09-03 09:13:24 +01:00
1 parent 6d0319fb63
commit f7cd94668f
9 files changed
+302 -49

No files matched your search

+20 -3
View File
@@ -42,6 +42,7 @@ from application.upload_limits import (
enforce_parseable_attachment,
is_unsupported_upload_message,
UnsupportedUploadTypeError,
unsupported_upload_message,
upload_limit_message,
UploadTooLargeError,
)
@@ -128,13 +129,27 @@ def _enforce_uploaded_audio_size_limit(file, filename: str) -> None:
enforce_audio_file_size_limit(size_bytes)
def _get_store_attachment_user_error(exc: Exception) -> str:
def _get_store_attachment_user_error(exc: Exception, filename: str | None = None) -> str:
"""Map an upload failure to a fixed client-facing message.
Every branch returns text this module composes itself. Nothing is read off
the exception, so no exception state — message or traceback — can reach a
response body (CodeQL py/stack-trace-exposure).
Args:
exc: The exception raised while processing one uploaded file.
filename: That file's name, which supplies the extension for the
unsupported-type message.
Returns:
The message to report for this failure.
"""
if isinstance(exc, AudioFileTooLargeError):
return build_stt_file_size_limit_message()
if isinstance(exc, UploadTooLargeError):
return upload_limit_message()
if isinstance(exc, UnsupportedUploadTypeError):
return str(exc)
return unsupported_upload_message(filename)
return "Failed to process file"
@@ -251,7 +266,9 @@ class StoreAttachment(Resource):
errors.append({
"upload_index": idx,
"filename": file.filename,
"error": _get_store_attachment_user_error(file_err),
"error": _get_store_attachment_user_error(
file_err, file.filename
),
})
if not tasks:
+11 -7
View File
@@ -28,10 +28,10 @@ SUPPORTED_SOURCE_EXTENSIONS = (
*SUPPORTED_AUDIO_EXTENSIONS,
)
# Suffixes the attachment path has a dedicated parser for — every key of
# ``get_default_file_extractor()`` plus ``.txt``, which needs none. Kept as a
# literal (importing ``bulk`` here would drag docling into the API process),
# with ``tests/parser/file/test_constants.py`` asserting the two agree.
# Suffixes the attachment path has a dedicated parser for — exactly the keys
# of ``get_default_file_extractor()``. Kept as a literal (importing ``bulk``
# here would drag docling into the API process), with
# ``tests/parser/file/test_constants.py`` asserting the two agree.
#
# This is *not* the whole attachment allow-list: a suffix with no parser is
# read by ``SimpleDirectoryReader``'s plain-text fallthrough, which is right
@@ -44,7 +44,7 @@ SUPPORTED_SOURCE_EXTENSIONS = (
# content check like any other binary.
#
# Mirrored in ``frontend/src/constants/fileUpload.ts``; update both together.
SUPPORTED_ATTACHMENT_EXTENSIONS = frozenset(
ATTACHMENT_PARSER_EXTENSIONS = frozenset(
{
*SUPPORTED_SOURCE_EXTENSIONS,
".xhtml",
@@ -57,6 +57,10 @@ SUPPORTED_ATTACHMENT_EXTENSIONS = frozenset(
".vtt",
".xml",
}
# .txt has no parser of its own — it *is* the plain-text fallthrough. It
# must be sniffed like any other unparsed suffix, or renaming a video to
# notes.txt walks straight back into the bug this gate exists for.
- {".txt"}
)
@@ -86,6 +90,6 @@ def has_attachment_parser(filename: str | None) -> bool:
filename: The upload's filename.
Returns:
True when the suffix is in ``SUPPORTED_ATTACHMENT_EXTENSIONS``.
True when the suffix is in ``ATTACHMENT_PARSER_EXTENSIONS``.
"""
return attachment_extension(filename) in SUPPORTED_ATTACHMENT_EXTENSIONS
return attachment_extension(filename) in ATTACHMENT_PARSER_EXTENSIONS
+16 -5
View File
@@ -64,6 +64,14 @@ _TEXT_SNIFF_BYTES = 8192
_MAX_NONTEXT_RATIO = 0.10
# Control bytes that occur in ordinary text: tab, LF, VT, FF, CR, ESC.
_TEXT_CONTROL_BYTES = frozenset({0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x1B})
# A UTF-16/32 file is half NUL bytes, so it has to be recognised by its BOM
# before the NUL test — Notepad's "Unicode" .txt is ordinary text.
_TEXT_BOMS = (
b"\xef\xbb\xbf",
b"\xff\xfe",
b"\xfe\xff",
b"\x00\x00\xfe\xff",
)
def looks_like_text(sample: bytes) -> bool:
@@ -74,10 +82,12 @@ def looks_like_text(sample: bytes) -> bool:
Returns:
False when the sample holds a NUL byte or too many other non-text
control bytes, True otherwise.
control bytes, True otherwise. A Unicode BOM settles it as text.
"""
if not sample:
return True
if sample.startswith(_TEXT_BOMS):
return True
if b"\x00" in sample:
return False
nontext = sum(
@@ -110,10 +120,11 @@ def enforce_parseable_attachment(
) -> None:
"""Reject an attachment that no parser handles and that is not plain text.
Suffixes with a parser (``SUPPORTED_ATTACHMENT_EXTENSIONS``) are admitted
unconditionally — a PDF is binary and parses fine. Everything else has to
read as text, which is what keeps a video or an archive out of the
plain-text fallthrough while leaving source, config and log files in.
Suffixes with a parser (``ATTACHMENT_PARSER_EXTENSIONS``) are admitted
unconditionally — a PDF is binary and parses fine. Everything else, .txt
included, has to read as text: that is what keeps a video or an archive
out of the plain-text fallthrough while leaving source, config and log
files in, whatever the file happens to be named.
Args:
path: Local path to the staged upload, readable before it is stored.