mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-14 12:16:20 +00:00
fix(attachments): close the .txt gap in the parseability gate
Review follow-ups on the attachment gate. .txt was listed as parser-backed, but it has no parser — it *is* the plain-text fallthrough. That let it skip the content check, so renaming a video to notes.txt walked straight back into the bug the gate exists for (verified: 5132 chars of binary "extracted" and stored). The list is now exactly the file extractor's keys, .txt included in the content check like any other unparsed suffix, and the drift test asserts equality rather than containment. The sniff now recognises a UTF-16/32 BOM as text, so a Notepad "Unicode" .txt is not caught by the NUL-byte rule. The picker's accept filter listed parser-backed suffixes only, hiding .txt, .py and .log — files the gate reads happily — behind "All files". It now carries text/* as well, so it can never be narrower than what the upload accepts. A rejected batch carries one errors entry per file, but the non-200 branch applied the top-level message to every chip, so two files failing for two reasons both reported the first one. Reasons are now matched by upload_index, with the top-level message as fallback. _get_store_attachment_user_error no longer reads str(exc): the unsupported-type message is rebuilt from the filename, so no exception state can reach a response body (CodeQL py/stack-trace-exposure).
This commit is contained in:
1 parent
6d0319fb63
commit
f7cd94668f
9 files changed
+302
-49
No files matched your search
@@ -42,6 +42,7 @@ from application.upload_limits import (
|
||||
enforce_parseable_attachment,
|
||||
is_unsupported_upload_message,
|
||||
UnsupportedUploadTypeError,
|
||||
unsupported_upload_message,
|
||||
upload_limit_message,
|
||||
UploadTooLargeError,
|
||||
)
|
||||
@@ -128,13 +129,27 @@ def _enforce_uploaded_audio_size_limit(file, filename: str) -> None:
|
||||
enforce_audio_file_size_limit(size_bytes)
|
||||
|
||||
|
||||
def _get_store_attachment_user_error(exc: Exception) -> str:
|
||||
def _get_store_attachment_user_error(exc: Exception, filename: str | None = None) -> str:
|
||||
"""Map an upload failure to a fixed client-facing message.
|
||||
|
||||
Every branch returns text this module composes itself. Nothing is read off
|
||||
the exception, so no exception state — message or traceback — can reach a
|
||||
response body (CodeQL py/stack-trace-exposure).
|
||||
|
||||
Args:
|
||||
exc: The exception raised while processing one uploaded file.
|
||||
filename: That file's name, which supplies the extension for the
|
||||
unsupported-type message.
|
||||
|
||||
Returns:
|
||||
The message to report for this failure.
|
||||
"""
|
||||
if isinstance(exc, AudioFileTooLargeError):
|
||||
return build_stt_file_size_limit_message()
|
||||
if isinstance(exc, UploadTooLargeError):
|
||||
return upload_limit_message()
|
||||
if isinstance(exc, UnsupportedUploadTypeError):
|
||||
return str(exc)
|
||||
return unsupported_upload_message(filename)
|
||||
return "Failed to process file"
|
||||
|
||||
|
||||
@@ -251,7 +266,9 @@ class StoreAttachment(Resource):
|
||||
errors.append({
|
||||
"upload_index": idx,
|
||||
"filename": file.filename,
|
||||
"error": _get_store_attachment_user_error(file_err),
|
||||
"error": _get_store_attachment_user_error(
|
||||
file_err, file.filename
|
||||
),
|
||||
})
|
||||
|
||||
if not tasks:
|
||||
|
||||
@@ -28,10 +28,10 @@ SUPPORTED_SOURCE_EXTENSIONS = (
|
||||
*SUPPORTED_AUDIO_EXTENSIONS,
|
||||
)
|
||||
|
||||
# Suffixes the attachment path has a dedicated parser for — every key of
|
||||
# ``get_default_file_extractor()`` plus ``.txt``, which needs none. Kept as a
|
||||
# literal (importing ``bulk`` here would drag docling into the API process),
|
||||
# with ``tests/parser/file/test_constants.py`` asserting the two agree.
|
||||
# Suffixes the attachment path has a dedicated parser for — exactly the keys
|
||||
# of ``get_default_file_extractor()``. Kept as a literal (importing ``bulk``
|
||||
# here would drag docling into the API process), with
|
||||
# ``tests/parser/file/test_constants.py`` asserting the two agree.
|
||||
#
|
||||
# This is *not* the whole attachment allow-list: a suffix with no parser is
|
||||
# read by ``SimpleDirectoryReader``'s plain-text fallthrough, which is right
|
||||
@@ -44,7 +44,7 @@ SUPPORTED_SOURCE_EXTENSIONS = (
|
||||
# content check like any other binary.
|
||||
#
|
||||
# Mirrored in ``frontend/src/constants/fileUpload.ts``; update both together.
|
||||
SUPPORTED_ATTACHMENT_EXTENSIONS = frozenset(
|
||||
ATTACHMENT_PARSER_EXTENSIONS = frozenset(
|
||||
{
|
||||
*SUPPORTED_SOURCE_EXTENSIONS,
|
||||
".xhtml",
|
||||
@@ -57,6 +57,10 @@ SUPPORTED_ATTACHMENT_EXTENSIONS = frozenset(
|
||||
".vtt",
|
||||
".xml",
|
||||
}
|
||||
# .txt has no parser of its own — it *is* the plain-text fallthrough. It
|
||||
# must be sniffed like any other unparsed suffix, or renaming a video to
|
||||
# notes.txt walks straight back into the bug this gate exists for.
|
||||
- {".txt"}
|
||||
)
|
||||
|
||||
|
||||
@@ -86,6 +90,6 @@ def has_attachment_parser(filename: str | None) -> bool:
|
||||
filename: The upload's filename.
|
||||
|
||||
Returns:
|
||||
True when the suffix is in ``SUPPORTED_ATTACHMENT_EXTENSIONS``.
|
||||
True when the suffix is in ``ATTACHMENT_PARSER_EXTENSIONS``.
|
||||
"""
|
||||
return attachment_extension(filename) in SUPPORTED_ATTACHMENT_EXTENSIONS
|
||||
return attachment_extension(filename) in ATTACHMENT_PARSER_EXTENSIONS
|
||||
@@ -64,6 +64,14 @@ _TEXT_SNIFF_BYTES = 8192
|
||||
_MAX_NONTEXT_RATIO = 0.10
|
||||
# Control bytes that occur in ordinary text: tab, LF, VT, FF, CR, ESC.
|
||||
_TEXT_CONTROL_BYTES = frozenset({0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x1B})
|
||||
# A UTF-16/32 file is half NUL bytes, so it has to be recognised by its BOM
|
||||
# before the NUL test — Notepad's "Unicode" .txt is ordinary text.
|
||||
_TEXT_BOMS = (
|
||||
b"\xef\xbb\xbf",
|
||||
b"\xff\xfe",
|
||||
b"\xfe\xff",
|
||||
b"\x00\x00\xfe\xff",
|
||||
)
|
||||
|
||||
|
||||
def looks_like_text(sample: bytes) -> bool:
|
||||
@@ -74,10 +82,12 @@ def looks_like_text(sample: bytes) -> bool:
|
||||
|
||||
Returns:
|
||||
False when the sample holds a NUL byte or too many other non-text
|
||||
control bytes, True otherwise.
|
||||
control bytes, True otherwise. A Unicode BOM settles it as text.
|
||||
"""
|
||||
if not sample:
|
||||
return True
|
||||
if sample.startswith(_TEXT_BOMS):
|
||||
return True
|
||||
if b"\x00" in sample:
|
||||
return False
|
||||
nontext = sum(
|
||||
@@ -110,10 +120,11 @@ def enforce_parseable_attachment(
|
||||
) -> None:
|
||||
"""Reject an attachment that no parser handles and that is not plain text.
|
||||
|
||||
Suffixes with a parser (``SUPPORTED_ATTACHMENT_EXTENSIONS``) are admitted
|
||||
unconditionally — a PDF is binary and parses fine. Everything else has to
|
||||
read as text, which is what keeps a video or an archive out of the
|
||||
plain-text fallthrough while leaving source, config and log files in.
|
||||
Suffixes with a parser (``ATTACHMENT_PARSER_EXTENSIONS``) are admitted
|
||||
unconditionally — a PDF is binary and parses fine. Everything else, .txt
|
||||
included, has to read as text: that is what keeps a video or an archive
|
||||
out of the plain-text fallthrough while leaving source, config and log
|
||||
files in, whatever the file happens to be named.
|
||||
|
||||
Args:
|
||||
path: Local path to the staged upload, readable before it is stored.
|
||||
|
||||
Reference in new issue
Block a user