mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-14 20:16:48 +00:00
fix(attachments): close the .txt gap in the parseability gate
Review follow-ups on the attachment gate. .txt was listed as parser-backed, but it has no parser — it *is* the plain-text fallthrough. That let it skip the content check, so renaming a video to notes.txt walked straight back into the bug the gate exists for (verified: 5132 chars of binary "extracted" and stored). The list is now exactly the file extractor's keys, .txt included in the content check like any other unparsed suffix, and the drift test asserts equality rather than containment. The sniff now recognises a UTF-16/32 BOM as text, so a Notepad "Unicode" .txt is not caught by the NUL-byte rule. The picker's accept filter listed parser-backed suffixes only, hiding .txt, .py and .log — files the gate reads happily — behind "All files". It now carries text/* as well, so it can never be narrower than what the upload accepts. A rejected batch carries one errors entry per file, but the non-200 branch applied the top-level message to every chip, so two files failing for two reasons both reported the first one. Reasons are now matched by upload_index, with the top-level message as fallback. _get_store_attachment_user_error no longer reads str(exc): the unsupported-type message is rebuilt from the filename, so no exception state can reach a response body (CodeQL py/stack-trace-exposure).
This commit is contained in:
1 parent
6d0319fb63
commit
f7cd94668f
9 files changed
+302
-49
No files matched your search
@@ -1,15 +1,16 @@
|
||||
"""Tests for the shared file-extension constants."""
|
||||
|
||||
import re
|
||||
from importlib.util import find_spec
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from application.parser.file.bulk import get_default_file_extractor
|
||||
from application.parser.file.constants import (
|
||||
ATTACHMENT_PARSER_EXTENSIONS,
|
||||
attachment_extension,
|
||||
has_attachment_parser,
|
||||
SUPPORTED_ATTACHMENT_EXTENSIONS,
|
||||
SUPPORTED_SOURCE_EXTENSIONS,
|
||||
)
|
||||
|
||||
@@ -19,30 +20,38 @@ FRONTEND_CONSTANTS = (
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_every_parser_backed_suffix_is_an_allowed_attachment():
|
||||
"""The allow-list is a hand-kept literal; this is what keeps it honest.
|
||||
def test_parser_extensions_match_the_extractor():
|
||||
"""The list is a hand-kept literal; this is what keeps it honest.
|
||||
|
||||
``SUPPORTED_ATTACHMENT_EXTENSIONS`` cannot import the extractor (that
|
||||
would pull docling into the API process), so a parser added to
|
||||
``ATTACHMENT_PARSER_EXTENSIONS`` cannot import the extractor (that would
|
||||
pull docling into the API process), so a parser added to
|
||||
``get_default_file_extractor`` would otherwise stay refused as an
|
||||
attachment — which is how .webp, .tiff, .vtt and .xml were first missed.
|
||||
A suffix listed here but *without* a parser is the opposite failure: it
|
||||
skips the content check and reaches the plain-text fallthrough, which is
|
||||
how a binary named notes.txt got through.
|
||||
"""
|
||||
parser_suffixes = set(get_default_file_extractor())
|
||||
|
||||
assert parser_suffixes <= set(SUPPORTED_ATTACHMENT_EXTENSIONS), (
|
||||
assert parser_suffixes <= set(ATTACHMENT_PARSER_EXTENSIONS), (
|
||||
"parsers exist for suffixes the attachment gate refuses: "
|
||||
f"{sorted(parser_suffixes - set(SUPPORTED_ATTACHMENT_EXTENSIONS))}"
|
||||
f"{sorted(parser_suffixes - set(ATTACHMENT_PARSER_EXTENSIONS))}"
|
||||
)
|
||||
if find_spec("docling") is not None:
|
||||
assert parser_suffixes == set(ATTACHMENT_PARSER_EXTENSIONS), (
|
||||
"listed as parser-backed but nothing parses them: "
|
||||
f"{sorted(set(ATTACHMENT_PARSER_EXTENSIONS) - parser_suffixes)}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_frontend_mirrors_the_attachment_allow_list():
|
||||
def test_frontend_mirrors_the_parser_extensions():
|
||||
"""The composer gates uploads client-side; a divergent list refuses valid files.
|
||||
|
||||
``ATTACHMENT_PARSER_EXTENSIONS`` in the frontend is a hand-kept copy of
|
||||
``SUPPORTED_ATTACHMENT_EXTENSIONS``. Nothing but this test connects them,
|
||||
and .mdx, .xhtml and .adoc were blocked in the UI while the API accepted
|
||||
them before it existed.
|
||||
the backend's. Nothing but this test connects them, and .mdx, .xhtml and
|
||||
.adoc were blocked in the UI while the API accepted them before it
|
||||
existed.
|
||||
"""
|
||||
if not FRONTEND_CONSTANTS.exists():
|
||||
pytest.skip("frontend sources not present in this checkout")
|
||||
@@ -56,16 +65,19 @@ def test_frontend_mirrors_the_attachment_allow_list():
|
||||
assert match, "ATTACHMENT_PARSER_EXTENSIONS not found in fileUpload.ts"
|
||||
frontend_extensions = set(re.findall(r"'(\.[^']+)'", match.group(1)))
|
||||
|
||||
assert frontend_extensions == set(SUPPORTED_ATTACHMENT_EXTENSIONS), (
|
||||
assert frontend_extensions == set(ATTACHMENT_PARSER_EXTENSIONS), (
|
||||
"frontend and backend attachment lists disagree — "
|
||||
f"backend only: {sorted(set(SUPPORTED_ATTACHMENT_EXTENSIONS) - frontend_extensions)}, "
|
||||
f"frontend only: {sorted(frontend_extensions - set(SUPPORTED_ATTACHMENT_EXTENSIONS))}"
|
||||
f"backend only: {sorted(set(ATTACHMENT_PARSER_EXTENSIONS) - frontend_extensions)}, "
|
||||
f"frontend only: {sorted(frontend_extensions - set(ATTACHMENT_PARSER_EXTENSIONS))}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_attachments_accept_everything_source_ingestion_does():
|
||||
assert set(SUPPORTED_SOURCE_EXTENSIONS) <= set(SUPPORTED_ATTACHMENT_EXTENSIONS)
|
||||
def test_source_ingestion_types_are_parser_backed_apart_from_text():
|
||||
"""Source ingestion's own list must not smuggle a parserless suffix past the sniff."""
|
||||
assert set(SUPPORTED_SOURCE_EXTENSIONS) - set(ATTACHMENT_PARSER_EXTENSIONS) == {
|
||||
".txt"
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@@ -93,3 +105,5 @@ def test_has_attachment_parser_is_case_insensitive_and_excludes_zip():
|
||||
assert not has_attachment_parser("archive.zip")
|
||||
assert not has_attachment_parser("main.py")
|
||||
assert not has_attachment_parser("clip.mp4")
|
||||
# .txt is the plain-text fallthrough itself, not a parser.
|
||||
assert not has_attachment_parser("notes.txt")
|
||||
Reference in new issue
Block a user