mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-15 08:16:38 +00:00
fix(attachments): refuse unparseable chat attachments
A chat attachment with no parser fell through to SimpleDirectoryReader's plain-text open(), so a phone-uploaded video was "extracted" into megabytes of binary garbage, truncated, and stored with extraction.status == "ok". Gate attachments in two tiers instead. A suffix with a dedicated parser is admitted on its name — a PDF is binary and parses fine. Anything else has to read as text: the first 8KB are sampled and refused on a NUL byte or too many other control bytes. That keeps source, config and log files working through the plain-text fallthrough, and keeps out videos, archives and renamed binaries alike. The route checks the staged spool before anything is stored or queued; the worker repeats the check where the local file exists, raising the non-retryable AttachmentRejectedError. SUPPORTED_ATTACHMENT_EXTENSIONS gains the parser-backed suffixes it was missing (.tiff, .tif, .bmp, .webp, .vtt, .xml) and is now exactly the file extractor's keys plus .txt, with a test asserting the two agree. The composer applies the same rule client-side, so an unsupported file is named before it costs an upload, and a test pins the frontend list to the backend one. Attachment failures now show their reason inline under the chips rather than only in a hover tooltip, which a touch user can never see, and only after a send was attempted. Dropped `accept` from the dropzone: it discarded rejected drops with no feedback and disagreed with the server about text files.
This commit is contained in:
1 parent
3e12a6bf62
commit
96cad1f8e2
15 files changed
+1079
-99
No files matched your search
@@ -39,6 +39,9 @@ from application.stt.stt_creator import STTCreator
|
||||
from application.tts.tts_creator import TTSCreator
|
||||
from application.upload_limits import (
|
||||
copy_upload_to_path,
|
||||
enforce_parseable_attachment,
|
||||
is_unsupported_upload_message,
|
||||
UnsupportedUploadTypeError,
|
||||
upload_limit_message,
|
||||
UploadTooLargeError,
|
||||
)
|
||||
@@ -130,6 +133,8 @@ def _get_store_attachment_user_error(exc: Exception) -> str:
|
||||
return build_stt_file_size_limit_message()
|
||||
if isinstance(exc, UploadTooLargeError):
|
||||
return upload_limit_message()
|
||||
if isinstance(exc, UnsupportedUploadTypeError):
|
||||
return str(exc)
|
||||
return "Failed to process file"
|
||||
|
||||
|
||||
@@ -214,6 +219,10 @@ class StoreAttachment(Resource):
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
staged_path = os.path.join(temp_dir, original_filename)
|
||||
copy_upload_to_path(file, staged_path)
|
||||
# Refuse here — nothing is stored or queued yet — so a
|
||||
# video the picker let through never reaches the
|
||||
# worker's reader, which would open it as text.
|
||||
enforce_parseable_attachment(staged_path, original_filename)
|
||||
with open(staged_path, "rb") as staged_stream:
|
||||
staged_upload = FileStorage(
|
||||
stream=staged_stream,
|
||||
@@ -263,6 +272,22 @@ class StoreAttachment(Resource):
|
||||
),
|
||||
413,
|
||||
)
|
||||
if errors and all(
|
||||
is_unsupported_upload_message(error.get("error")) for error in errors
|
||||
):
|
||||
# Tell the user which type was refused rather than the
|
||||
# generic copy — this is the branch a phone picker that
|
||||
# ignores ``accept`` lands in.
|
||||
return make_response(
|
||||
jsonify(
|
||||
{
|
||||
"success": False,
|
||||
"message": errors[0]["error"],
|
||||
"errors": errors,
|
||||
}
|
||||
),
|
||||
400,
|
||||
)
|
||||
return make_response(
|
||||
jsonify({"status": "error", "message": "No valid files to upload"}),
|
||||
400,
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
"""Shared file-extension constants for parsing and ingestion flows."""
|
||||
|
||||
import os
|
||||
|
||||
from application.stt.constants import SUPPORTED_AUDIO_EXTENSIONS
|
||||
|
||||
|
||||
@@ -25,3 +27,65 @@ SUPPORTED_SOURCE_EXTENSIONS = (
|
||||
*SUPPORTED_SOURCE_IMAGE_EXTENSIONS,
|
||||
*SUPPORTED_AUDIO_EXTENSIONS,
|
||||
)
|
||||
|
||||
# Suffixes the attachment path has a dedicated parser for — every key of
|
||||
# ``get_default_file_extractor()`` plus ``.txt``, which needs none. Kept as a
|
||||
# literal (importing ``bulk`` here would drag docling into the API process),
|
||||
# with ``tests/parser/file/test_constants.py`` asserting the two agree.
|
||||
#
|
||||
# This is *not* the whole attachment allow-list: a suffix with no parser is
|
||||
# read by ``SimpleDirectoryReader``'s plain-text fallthrough, which is right
|
||||
# for a .py or a .log and catastrophic for a video — the reader "extracts"
|
||||
# megabytes of binary garbage, truncates it, and stores it with
|
||||
# ``extraction.status == "ok"``. So unparsed suffixes are admitted on
|
||||
# content instead (``upload_limits.enforce_parseable_attachment``): text
|
||||
# passes, binary is refused. Zip is deliberately absent — source ingestion
|
||||
# extracts archives, the attachment path does not, and a zip fails the
|
||||
# content check like any other binary.
|
||||
#
|
||||
# Mirrored in ``frontend/src/constants/fileUpload.ts``; update both together.
|
||||
SUPPORTED_ATTACHMENT_EXTENSIONS = frozenset(
|
||||
{
|
||||
*SUPPORTED_SOURCE_EXTENSIONS,
|
||||
".xhtml",
|
||||
".adoc",
|
||||
".asciidoc",
|
||||
".tiff",
|
||||
".tif",
|
||||
".bmp",
|
||||
".webp",
|
||||
".vtt",
|
||||
".xml",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def attachment_extension(filename: str | None) -> str:
|
||||
"""Return the lower-cased extension of ``filename`` including the dot, or ``""``.
|
||||
|
||||
Args:
|
||||
filename: A bare filename or path; ``None`` and empty strings yield ``""``.
|
||||
|
||||
Returns:
|
||||
The last suffix in lower case (``".pdf"``), or ``""`` when there is none.
|
||||
"""
|
||||
if not filename:
|
||||
return ""
|
||||
return os.path.splitext(os.path.basename(str(filename)))[1].lower()
|
||||
|
||||
|
||||
def has_attachment_parser(filename: str | None) -> bool:
|
||||
"""Return whether an attachment's suffix has a dedicated parser.
|
||||
|
||||
A False result does not mean the file is refused: it means nothing but the
|
||||
plain-text fallthrough will read it, so it has to earn its place on
|
||||
content. The decision is by extension only, never by the mime type a
|
||||
browser reports (mobile pickers ignore ``accept`` and lie).
|
||||
|
||||
Args:
|
||||
filename: The upload's filename.
|
||||
|
||||
Returns:
|
||||
True when the suffix is in ``SUPPORTED_ATTACHMENT_EXTENSIONS``.
|
||||
"""
|
||||
return attachment_extension(filename) in SUPPORTED_ATTACHMENT_EXTENSIONS
|
||||
@@ -8,6 +8,10 @@ from contextlib import suppress
|
||||
from typing import BinaryIO
|
||||
|
||||
from application.core.settings import settings
|
||||
from application.parser.file.constants import (
|
||||
attachment_extension,
|
||||
has_attachment_parser,
|
||||
)
|
||||
|
||||
|
||||
_COPY_CHUNK_BYTES = 64 * 1024
|
||||
@@ -26,6 +30,106 @@ class UploadTooLargeError(ValueError):
|
||||
"""Raised when one uploaded file exceeds the configured byte cap."""
|
||||
|
||||
|
||||
class UnsupportedUploadTypeError(ValueError):
|
||||
"""Raised when an uploaded attachment has a file type the worker cannot parse."""
|
||||
|
||||
|
||||
_UNSUPPORTED_UPLOAD_PREFIX = "Unsupported file type"
|
||||
|
||||
|
||||
def unsupported_upload_message(filename: str | None) -> str:
|
||||
"""Return the stable client-facing rejection message for an unparseable upload.
|
||||
|
||||
Args:
|
||||
filename: The rejected upload's filename.
|
||||
|
||||
Returns:
|
||||
``"Unsupported file type: .mp4"`` style text, naming the extension.
|
||||
"""
|
||||
extension = attachment_extension(filename)
|
||||
return f"{_UNSUPPORTED_UPLOAD_PREFIX}: {extension or '(no extension)'}"
|
||||
|
||||
|
||||
def is_unsupported_upload_message(message: str | None) -> bool:
|
||||
"""Return whether ``message`` is one produced by :func:`unsupported_upload_message`."""
|
||||
return bool(message) and str(message).startswith(_UNSUPPORTED_UPLOAD_PREFIX)
|
||||
|
||||
|
||||
# A suffix with no parser is read by ``SimpleDirectoryReader``'s plain-text
|
||||
# fallthrough, so it is admitted on content: enough of the head to recognise a
|
||||
# container header, and a tolerance that keeps real text (UTF-8 accents, an
|
||||
# ANSI-coloured log) in while a random binary — ~12.5% of bytes below 0x20 —
|
||||
# stays out.
|
||||
_TEXT_SNIFF_BYTES = 8192
|
||||
_MAX_NONTEXT_RATIO = 0.10
|
||||
# Control bytes that occur in ordinary text: tab, LF, VT, FF, CR, ESC.
|
||||
_TEXT_CONTROL_BYTES = frozenset({0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x1B})
|
||||
|
||||
|
||||
def looks_like_text(sample: bytes) -> bool:
|
||||
"""Return whether a leading byte sample reads as text rather than binary.
|
||||
|
||||
Args:
|
||||
sample: The first bytes of a file; an empty sample counts as text.
|
||||
|
||||
Returns:
|
||||
False when the sample holds a NUL byte or too many other non-text
|
||||
control bytes, True otherwise.
|
||||
"""
|
||||
if not sample:
|
||||
return True
|
||||
if b"\x00" in sample:
|
||||
return False
|
||||
nontext = sum(
|
||||
1
|
||||
for byte in sample
|
||||
if (byte < 0x20 and byte not in _TEXT_CONTROL_BYTES) or byte == 0x7F
|
||||
)
|
||||
return nontext / len(sample) <= _MAX_NONTEXT_RATIO
|
||||
|
||||
|
||||
def file_looks_like_text(path: str | os.PathLike[str]) -> bool:
|
||||
"""Return whether a file on disk reads as text, by its leading bytes.
|
||||
|
||||
Args:
|
||||
path: Filesystem path to sample.
|
||||
|
||||
Returns:
|
||||
The :func:`looks_like_text` verdict for the file's head; True when the
|
||||
file cannot be read, leaving that failure to the parser to report.
|
||||
"""
|
||||
try:
|
||||
with open(path, "rb") as handle:
|
||||
return looks_like_text(handle.read(_TEXT_SNIFF_BYTES))
|
||||
except OSError:
|
||||
return True
|
||||
|
||||
|
||||
def enforce_parseable_attachment(
|
||||
path: str | os.PathLike[str], filename: str | None
|
||||
) -> None:
|
||||
"""Reject an attachment that no parser handles and that is not plain text.
|
||||
|
||||
Suffixes with a parser (``SUPPORTED_ATTACHMENT_EXTENSIONS``) are admitted
|
||||
unconditionally — a PDF is binary and parses fine. Everything else has to
|
||||
read as text, which is what keeps a video or an archive out of the
|
||||
plain-text fallthrough while leaving source, config and log files in.
|
||||
|
||||
Args:
|
||||
path: Local path to the staged upload, readable before it is stored.
|
||||
filename: The upload's filename, used for the suffix and the message.
|
||||
|
||||
Raises:
|
||||
UnsupportedUploadTypeError: When the file has no parser and its
|
||||
contents are binary.
|
||||
"""
|
||||
if has_attachment_parser(filename):
|
||||
return
|
||||
if file_looks_like_text(path):
|
||||
return
|
||||
raise UnsupportedUploadTypeError(unsupported_upload_message(filename))
|
||||
|
||||
|
||||
class _LimitedRawReader(io.RawIOBase):
|
||||
"""Expose a binary stream while enforcing an exact cumulative read cap."""
|
||||
|
||||
|
||||
@@ -55,6 +55,10 @@ from application.storage.db.repositories.wiki_pages import (
|
||||
from application.storage.db.session import db_readonly, db_session
|
||||
from application.storage.db.source_config import SourceConfig
|
||||
from application.storage.storage_creator import StorageCreator
|
||||
from application.upload_limits import (
|
||||
enforce_parseable_attachment,
|
||||
UnsupportedUploadTypeError,
|
||||
)
|
||||
from application.utils import (
|
||||
count_tokens_docs,
|
||||
get_encoding,
|
||||
@@ -1554,6 +1558,26 @@ class AttachmentRejectedError(Exception):
|
||||
"""
|
||||
|
||||
|
||||
def _reject_unparseable_attachment(local_path: str, filename: str) -> None:
|
||||
"""Reject an attachment with no parser whose contents are binary.
|
||||
|
||||
Defence in depth behind the route's gate: ``SimpleDirectoryReader`` opens
|
||||
any suffix it has no parser for as plain text, so a binary must fail here,
|
||||
visibly, rather than "succeed" as garbage.
|
||||
|
||||
Args:
|
||||
local_path: Filesystem path of the attachment about to be parsed.
|
||||
filename: The upload's original filename, which carries the suffix.
|
||||
|
||||
Raises:
|
||||
AttachmentRejectedError: If the file has no parser and is not text.
|
||||
"""
|
||||
try:
|
||||
enforce_parseable_attachment(local_path, filename)
|
||||
except UnsupportedUploadTypeError as exc:
|
||||
raise AttachmentRejectedError(str(exc)) from exc
|
||||
|
||||
|
||||
def _reject_attachment_zip_bomb(local_path: str) -> None:
|
||||
"""Reject a zip-container attachment that decompresses to too much.
|
||||
|
||||
@@ -1751,6 +1775,7 @@ def attachment_worker(self, file_info, user):
|
||||
parser_name = type(_parser).__name__ if _parser is not None else "SimpleDirectoryReader"
|
||||
|
||||
def _parse_local_file(local_path: str, **kwargs) -> Document:
|
||||
_reject_unparseable_attachment(local_path, filename)
|
||||
_reject_attachment_zip_bomb(local_path)
|
||||
parse_path, is_temp_copy = _bounded_attachment_copy(local_path)
|
||||
try:
|
||||
|
||||
@@ -48,7 +48,9 @@ import { useArmedSend } from './message-input/armedSend';
|
||||
import { handleAbort } from '../conversation/conversationSlice';
|
||||
import {
|
||||
AUDIO_FILE_ACCEPT_ATTR,
|
||||
FILE_UPLOAD_ACCEPT,
|
||||
getFileExtension,
|
||||
parseUploadErrorMessage,
|
||||
partitionAttachmentFiles,
|
||||
} from '../constants/fileUpload';
|
||||
import { UserToolType } from '../settings/types';
|
||||
import { isChatToolVisible } from '../utils/toolUtils';
|
||||
@@ -497,8 +499,31 @@ export default function MessageInput({
|
||||
);
|
||||
|
||||
const uploadFiles = useCallback(
|
||||
(files: File[]) => {
|
||||
if (!files || files.length === 0) return;
|
||||
async (incomingFiles: File[]) => {
|
||||
if (!incomingFiles || incomingFiles.length === 0) return;
|
||||
|
||||
// Run the server's own rule here, not just the input's `accept`:
|
||||
// mobile pickers ignore `accept`, and a file the server will refuse
|
||||
// should say so before it costs an upload. Surface the refusal as a
|
||||
// failed chip so the user sees why instead of a silent drop.
|
||||
const { supported, unsupported } =
|
||||
await partitionAttachmentFiles(incomingFiles);
|
||||
unsupported.forEach((file) => {
|
||||
dispatch(
|
||||
addAttachment({
|
||||
id: generateId(),
|
||||
fileName: file.name,
|
||||
progress: 0,
|
||||
status: 'failed' as const,
|
||||
taskId: '',
|
||||
errorMessage: t('conversation.attachments.unsupportedType', {
|
||||
extension: getFileExtension(file.name) || '?',
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
if (supported.length === 0) return;
|
||||
const files = supported;
|
||||
|
||||
const apiHost = import.meta.env.VITE_API_HOST;
|
||||
|
||||
@@ -579,9 +604,14 @@ export default function MessageInput({
|
||||
tasksByIndex.set(uploadIndex, task);
|
||||
});
|
||||
|
||||
const errorsByIndex = new Map<number, string | undefined>();
|
||||
errors.forEach((errorItem) => {
|
||||
if (typeof errorItem.upload_index === 'number') {
|
||||
failedIndices.add(errorItem.upload_index);
|
||||
errorsByIndex.set(
|
||||
errorItem.upload_index,
|
||||
errorItem.error,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -616,7 +646,10 @@ export default function MessageInput({
|
||||
dispatch(
|
||||
updateAttachment({
|
||||
id: uiId,
|
||||
updates: { status: 'failed' },
|
||||
updates: {
|
||||
status: 'failed',
|
||||
errorMessage: errorsByIndex.get(index),
|
||||
},
|
||||
}),
|
||||
);
|
||||
return;
|
||||
@@ -748,11 +781,12 @@ export default function MessageInput({
|
||||
}
|
||||
} else {
|
||||
console.error('Upload failed', status, xhr.responseText);
|
||||
const errorMessage = parseUploadErrorMessage(xhr.responseText);
|
||||
Object.values(indexToUiId).forEach((id) =>
|
||||
dispatch(
|
||||
updateAttachment({
|
||||
id,
|
||||
updates: { status: 'failed' },
|
||||
updates: { status: 'failed', errorMessage },
|
||||
}),
|
||||
),
|
||||
);
|
||||
@@ -871,7 +905,10 @@ export default function MessageInput({
|
||||
dispatch(
|
||||
updateAttachment({
|
||||
id: uniqueId,
|
||||
updates: { status: 'failed' },
|
||||
updates: {
|
||||
status: 'failed',
|
||||
errorMessage: parseUploadErrorMessage(xhr.responseText),
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
@@ -891,7 +928,7 @@ export default function MessageInput({
|
||||
xhr.send(formData);
|
||||
});
|
||||
},
|
||||
[dispatch, token, trackAttachment],
|
||||
[dispatch, t, token, trackAttachment],
|
||||
);
|
||||
|
||||
const handleFileAttachment = (e: React.ChangeEvent<HTMLInputElement>) => {
|
||||
@@ -928,7 +965,10 @@ export default function MessageInput({
|
||||
setHandleDragActive(false);
|
||||
},
|
||||
maxSize: 25000000,
|
||||
accept: FILE_UPLOAD_ACCEPT,
|
||||
// No `accept`: react-dropzone would drop a rejected file on the floor
|
||||
// with no feedback, and its mime matching disagrees with the server for
|
||||
// text files that have no parser (.py, .log). uploadFiles applies the
|
||||
// server's rule and reports what it refuses.
|
||||
});
|
||||
|
||||
const handleInput = useCallback(() => {
|
||||
@@ -1594,7 +1634,13 @@ export default function MessageInput({
|
||||
onChange={handleVoiceFileAttachment}
|
||||
/>
|
||||
|
||||
<div className="border-border bg-card relative flex w-full flex-col rounded-3xl border dark:bg-transparent">
|
||||
{/* translate="no": keep Chrome's page translator out of the composer —
|
||||
it rewrites text nodes into <font> wrappers and React loses the
|
||||
controls (dead Attach button, see AttachFileButton). */}
|
||||
<div
|
||||
translate="no"
|
||||
className="border-border bg-card relative flex w-full flex-col rounded-3xl border dark:bg-transparent"
|
||||
>
|
||||
<AttachmentChipList
|
||||
attachments={attachments}
|
||||
draggingId={draggingId}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { useTranslation } from 'react-i18next';
|
||||
|
||||
import ClipIcon from '../../assets/clip.svg';
|
||||
import { FILE_UPLOAD_ACCEPT_ATTR } from '../../constants/fileUpload';
|
||||
import { ATTACHMENT_FILE_ACCEPT_ATTR } from '../../constants/fileUpload';
|
||||
|
||||
type AttachFileButtonProps = {
|
||||
onChange: (e: React.ChangeEvent<HTMLInputElement>) => void;
|
||||
@@ -11,7 +11,14 @@ export default function AttachFileButton({ onChange }: AttachFileButtonProps) {
|
||||
const { t } = useTranslation();
|
||||
|
||||
return (
|
||||
<label className="xs:px-3 xs:py-1.5 dark:border-border border-border hover:bg-muted dark:hover:bg-muted flex cursor-pointer items-center rounded-full border px-2 py-1 transition-colors">
|
||||
// translate="no": Chrome's page translator rewrites text nodes inside
|
||||
// this label into <font> wrappers and React then loses the control — a
|
||||
// user with auto-translate on rage-clicked a dead Attach button until
|
||||
// they reloaded. Keep the composer's controls out of the translator.
|
||||
<label
|
||||
translate="no"
|
||||
className="xs:px-3 xs:py-1.5 dark:border-border border-border hover:bg-muted dark:hover:bg-muted flex cursor-pointer items-center rounded-full border px-2 py-1 transition-colors"
|
||||
>
|
||||
<img
|
||||
src={ClipIcon}
|
||||
alt="Attach"
|
||||
@@ -24,7 +31,7 @@ export default function AttachFileButton({ onChange }: AttachFileButtonProps) {
|
||||
type="file"
|
||||
className="hidden"
|
||||
multiple
|
||||
accept={FILE_UPLOAD_ACCEPT_ATTR}
|
||||
accept={ATTACHMENT_FILE_ACCEPT_ATTR}
|
||||
onChange={onChange}
|
||||
/>
|
||||
</label>
|
||||
|
||||
@@ -25,94 +25,122 @@ export default function AttachmentChipList({
|
||||
}: AttachmentChipListProps) {
|
||||
const { t } = useTranslation();
|
||||
|
||||
// A tooltip is the one place a touch user can never look, and this list is
|
||||
// where a phone picker's unsupported file lands. Show the reason inline,
|
||||
// as soon as it is known, rather than only once a send is attempted.
|
||||
const failures = attachments.filter(
|
||||
(attachment) => attachment.status === 'failed' && attachment.errorMessage,
|
||||
);
|
||||
|
||||
return (
|
||||
<div className="flex flex-wrap gap-1.5 px-2 py-2 sm:gap-2 sm:px-3">
|
||||
{attachments.map((attachment) => {
|
||||
return (
|
||||
<div
|
||||
key={attachment.id}
|
||||
draggable={true}
|
||||
onDragStart={(e) => onDragStart(e, attachment.id)}
|
||||
onDragOver={onDragOver}
|
||||
onDrop={(e) => onDropOn(e, attachment.id)}
|
||||
className={`group dark:text-foreground bg-muted text-muted-foreground dark:bg-accent relative flex items-center rounded-xl px-2 py-1 text-xs sm:px-3 sm:py-1.5 sm:text-sm ${
|
||||
attachment.status !== 'completed' ? 'opacity-70' : 'opacity-100'
|
||||
} ${
|
||||
draggingId === attachment.id
|
||||
? 'ring-dashed opacity-60 ring-2 ring-purple-200'
|
||||
: ''
|
||||
}`}
|
||||
title={attachment.fileName}
|
||||
>
|
||||
<div className="bg-primary mr-2 flex h-8 w-8 items-center justify-center rounded-md p-1">
|
||||
{attachment.status === 'completed' && (
|
||||
<img
|
||||
src={DocumentationDark}
|
||||
alt="Attachment"
|
||||
className="h-[15px] w-[15px] object-fill"
|
||||
/>
|
||||
)}
|
||||
|
||||
{attachment.status === 'failed' && (
|
||||
<img
|
||||
src={AlertIcon}
|
||||
alt="Failed"
|
||||
className="h-[15px] w-[15px] object-fill"
|
||||
/>
|
||||
)}
|
||||
|
||||
{(attachment.status === 'uploading' ||
|
||||
attachment.status === 'processing') && (
|
||||
<div className="flex h-[15px] w-[15px] items-center justify-center">
|
||||
<svg className="h-[15px] w-[15px]" viewBox="0 0 24 24">
|
||||
<circle
|
||||
className="opacity-0"
|
||||
cx="12"
|
||||
cy="12"
|
||||
r="10"
|
||||
stroke="transparent"
|
||||
strokeWidth="4"
|
||||
fill="none"
|
||||
/>
|
||||
<circle
|
||||
className="text-[#ECECF1]"
|
||||
cx="12"
|
||||
cy="12"
|
||||
r="10"
|
||||
stroke="currentColor"
|
||||
strokeWidth="4"
|
||||
fill="none"
|
||||
strokeDasharray="62.83"
|
||||
strokeDashoffset={62.83 * (1 - attachment.progress / 100)}
|
||||
transform="rotate(-90 12 12)"
|
||||
/>
|
||||
</svg>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<span className="max-w-[120px] truncate font-medium sm:max-w-[150px]">
|
||||
{attachment.fileName}
|
||||
</span>
|
||||
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
className="ml-1.5 h-auto w-auto rounded-full p-1"
|
||||
onClick={() => {
|
||||
onRemove(attachment.id);
|
||||
}}
|
||||
aria-label={t('conversation.attachments.remove')}
|
||||
<>
|
||||
<div className="flex flex-wrap gap-1.5 px-2 py-2 sm:gap-2 sm:px-3">
|
||||
{attachments.map((attachment) => {
|
||||
return (
|
||||
<div
|
||||
key={attachment.id}
|
||||
draggable={true}
|
||||
onDragStart={(e) => onDragStart(e, attachment.id)}
|
||||
onDragOver={onDragOver}
|
||||
onDrop={(e) => onDropOn(e, attachment.id)}
|
||||
className={`group dark:text-foreground bg-muted text-muted-foreground dark:bg-accent relative flex items-center rounded-xl px-2 py-1 text-xs sm:px-3 sm:py-1.5 sm:text-sm ${
|
||||
attachment.status !== 'completed' ? 'opacity-70' : 'opacity-100'
|
||||
} ${
|
||||
draggingId === attachment.id
|
||||
? 'ring-dashed opacity-60 ring-2 ring-purple-200'
|
||||
: ''
|
||||
}`}
|
||||
title={
|
||||
attachment.status === 'failed' && attachment.errorMessage
|
||||
? `${attachment.fileName}: ${attachment.errorMessage}`
|
||||
: attachment.fileName
|
||||
}
|
||||
>
|
||||
<X
|
||||
<div className="bg-primary mr-2 flex h-8 w-8 items-center justify-center rounded-md p-1">
|
||||
{attachment.status === 'completed' && (
|
||||
<img
|
||||
src={DocumentationDark}
|
||||
alt="Attachment"
|
||||
className="h-[15px] w-[15px] object-fill"
|
||||
/>
|
||||
)}
|
||||
|
||||
{attachment.status === 'failed' && (
|
||||
<img
|
||||
src={AlertIcon}
|
||||
alt="Failed"
|
||||
className="h-[15px] w-[15px] object-fill"
|
||||
/>
|
||||
)}
|
||||
|
||||
{(attachment.status === 'uploading' ||
|
||||
attachment.status === 'processing') && (
|
||||
<div className="flex h-[15px] w-[15px] items-center justify-center">
|
||||
<svg className="h-[15px] w-[15px]" viewBox="0 0 24 24">
|
||||
<circle
|
||||
className="opacity-0"
|
||||
cx="12"
|
||||
cy="12"
|
||||
r="10"
|
||||
stroke="transparent"
|
||||
strokeWidth="4"
|
||||
fill="none"
|
||||
/>
|
||||
<circle
|
||||
className="text-[#ECECF1]"
|
||||
cx="12"
|
||||
cy="12"
|
||||
r="10"
|
||||
stroke="currentColor"
|
||||
strokeWidth="4"
|
||||
fill="none"
|
||||
strokeDasharray="62.83"
|
||||
strokeDashoffset={
|
||||
62.83 * (1 - attachment.progress / 100)
|
||||
}
|
||||
transform="rotate(-90 12 12)"
|
||||
/>
|
||||
</svg>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<span className="max-w-[120px] truncate font-medium sm:max-w-[150px]">
|
||||
{attachment.fileName}
|
||||
</span>
|
||||
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon-sm"
|
||||
className="ml-1.5 h-auto w-auto rounded-full p-1"
|
||||
onClick={() => {
|
||||
onRemove(attachment.id);
|
||||
}}
|
||||
aria-label={t('conversation.attachments.remove')}
|
||||
className="h-2.5 w-2.5"
|
||||
/>
|
||||
</Button>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
>
|
||||
<X
|
||||
aria-label={t('conversation.attachments.remove')}
|
||||
className="h-2.5 w-2.5"
|
||||
/>
|
||||
</Button>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
|
||||
{failures.length > 0 && (
|
||||
<div
|
||||
className="flex flex-col gap-0.5 px-2 pb-1 text-xs text-[#B42318] sm:px-3"
|
||||
role="alert"
|
||||
>
|
||||
{failures.map((attachment) => (
|
||||
<span key={attachment.id}>
|
||||
{attachment.fileName}: {attachment.errorMessage}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import {
|
||||
ATTACHMENT_FILE_ACCEPT_ATTR,
|
||||
ATTACHMENT_PARSER_EXTENSIONS,
|
||||
getFileExtension,
|
||||
hasAttachmentParser,
|
||||
looksLikeText,
|
||||
parseUploadErrorMessage,
|
||||
partitionAttachmentFiles,
|
||||
} from './fileUpload';
|
||||
|
||||
const file = (name: string, body: BlobPart = 'x', type = '') =>
|
||||
new File([body], name, { type });
|
||||
|
||||
const bytes = (...values: number[]) => new Uint8Array(values);
|
||||
|
||||
const MP4_HEADER = bytes(
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x18,
|
||||
0x66,
|
||||
0x74,
|
||||
0x79,
|
||||
0x70,
|
||||
0x69,
|
||||
0x73,
|
||||
0x6f,
|
||||
0x6d,
|
||||
);
|
||||
|
||||
describe('attachment type gate', () => {
|
||||
it('extracts a lower-cased extension', () => {
|
||||
expect(getFileExtension('Photo.JPG')).toBe('.jpg');
|
||||
expect(getFileExtension('a.tar.gz')).toBe('.gz');
|
||||
expect(getFileExtension('noext')).toBe('');
|
||||
expect(getFileExtension('.env')).toBe('');
|
||||
});
|
||||
|
||||
it('recognises parser-backed suffixes by name, case-insensitively', () => {
|
||||
expect(hasAttachmentParser(file('Report.PDF'))).toBe(true);
|
||||
expect(hasAttachmentParser(file('scan.WebP'))).toBe(true);
|
||||
expect(hasAttachmentParser(file('subs.vtt'))).toBe(true);
|
||||
// No parser — these are judged on content, not on the suffix.
|
||||
expect(hasAttachmentParser(file('main.py'))).toBe(false);
|
||||
expect(hasAttachmentParser(file('archive.zip'))).toBe(false);
|
||||
expect(hasAttachmentParser(file('clip.mp4'))).toBe(false);
|
||||
});
|
||||
|
||||
it('mirrors the backend list: no zip, no video, images and markup included', () => {
|
||||
const listed = ATTACHMENT_FILE_ACCEPT_ATTR.split(',');
|
||||
expect(listed).toEqual([...ATTACHMENT_PARSER_EXTENSIONS]);
|
||||
expect(listed).toContain('.pdf');
|
||||
// Parser-backed suffixes the first cut of this list missed.
|
||||
for (const ext of [
|
||||
'.webp',
|
||||
'.tiff',
|
||||
'.tif',
|
||||
'.bmp',
|
||||
'.vtt',
|
||||
'.xml',
|
||||
'.mdx',
|
||||
]) {
|
||||
expect(listed).toContain(ext);
|
||||
}
|
||||
expect(listed).not.toContain('.zip');
|
||||
expect(listed).not.toContain('.mp4');
|
||||
});
|
||||
});
|
||||
|
||||
describe('looksLikeText', () => {
|
||||
it('accepts empty, plain, accented and ANSI-coloured text', () => {
|
||||
expect(looksLikeText(new Uint8Array())).toBe(true);
|
||||
expect(looksLikeText(new TextEncoder().encode('plain text\n'))).toBe(true);
|
||||
expect(looksLikeText(new TextEncoder().encode('café — em dash\n'))).toBe(
|
||||
true,
|
||||
);
|
||||
expect(looksLikeText(bytes(0x1b, 0x5b, 0x33, 0x31, 0x6d, 0x6f, 0x6b))).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects a NUL byte and dense control bytes', () => {
|
||||
expect(looksLikeText(MP4_HEADER)).toBe(false);
|
||||
expect(looksLikeText(bytes(0x74, 0x78, 0x00, 0x74))).toBe(false);
|
||||
expect(
|
||||
looksLikeText(
|
||||
new Uint8Array(Array.from({ length: 64 }, (_, i) => i + 1)),
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('partitionAttachmentFiles', () => {
|
||||
it('refuses video and archives whatever the picker claims the type is', async () => {
|
||||
const { supported, unsupported } = await partitionAttachmentFiles([
|
||||
file('clip.mp4', MP4_HEADER, 'text/plain'),
|
||||
file('archive.zip', bytes(0x50, 0x4b, 0x03, 0x04, 0x00, 0x00)),
|
||||
]);
|
||||
expect(supported).toEqual([]);
|
||||
expect(unsupported.map((f) => f.name)).toEqual(['clip.mp4', 'archive.zip']);
|
||||
});
|
||||
|
||||
it('keeps text files that have no parser — the backend reads them', async () => {
|
||||
const { supported, unsupported } = await partitionAttachmentFiles([
|
||||
file('main.py', 'def main():\n return 1\n'),
|
||||
file('server.log', '2026-09-02 ERROR boom\n'),
|
||||
file('Dockerfile', 'FROM python:3.12\n'),
|
||||
]);
|
||||
expect(supported.map((f) => f.name)).toEqual([
|
||||
'main.py',
|
||||
'server.log',
|
||||
'Dockerfile',
|
||||
]);
|
||||
expect(unsupported).toEqual([]);
|
||||
});
|
||||
|
||||
it('admits parser-backed types on their suffix, binary contents and all', async () => {
|
||||
// A PDF is binary; the sniff must never be applied to it.
|
||||
const { supported, unsupported } = await partitionAttachmentFiles([
|
||||
file('paper.pdf', MP4_HEADER),
|
||||
file('photo.jpeg', MP4_HEADER),
|
||||
]);
|
||||
expect(supported.map((f) => f.name)).toEqual(['paper.pdf', 'photo.jpeg']);
|
||||
expect(unsupported).toEqual([]);
|
||||
});
|
||||
|
||||
it('partitions a mixed selection and keeps the original order', async () => {
|
||||
const mp4 = file('clip.mp4', MP4_HEADER);
|
||||
const txt = file('notes.txt');
|
||||
const pdf = file('paper.pdf');
|
||||
const { supported, unsupported } = await partitionAttachmentFiles([
|
||||
mp4,
|
||||
txt,
|
||||
pdf,
|
||||
]);
|
||||
expect(supported).toEqual([txt, pdf]);
|
||||
expect(unsupported).toEqual([mp4]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseUploadErrorMessage', () => {
|
||||
it('returns the server message when present', () => {
|
||||
expect(
|
||||
parseUploadErrorMessage(
|
||||
'{"success":false,"message":"Unsupported file type: .mp4"}',
|
||||
),
|
||||
).toBe('Unsupported file type: .mp4');
|
||||
});
|
||||
|
||||
it('returns undefined for non-JSON bodies or a missing message', () => {
|
||||
expect(parseUploadErrorMessage('<html>502</html>')).toBeUndefined();
|
||||
expect(parseUploadErrorMessage('{"ok":1}')).toBeUndefined();
|
||||
expect(parseUploadErrorMessage('')).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -80,3 +80,129 @@ export const SOURCE_FILE_TREE_ACCEPT_ATTR = [
|
||||
'.jpg',
|
||||
'.jpeg',
|
||||
].join(',');
|
||||
|
||||
/**
|
||||
* Chat-attachment suffixes with a dedicated parser. Mirrors the backend's
|
||||
* `SUPPORTED_ATTACHMENT_EXTENSIONS` (application/parser/file/constants.py) —
|
||||
* update both together. Zip is absent: source ingestion extracts archives,
|
||||
* the attachment path does not.
|
||||
*
|
||||
* Not the whole allow-list. A suffix that isn't here (.py, .log, .yaml) is
|
||||
* read by the backend's plain-text fallthrough, so it is judged on content
|
||||
* by `partitionAttachmentFiles` instead, exactly as the server does.
|
||||
*/
|
||||
export const ATTACHMENT_PARSER_EXTENSIONS: readonly string[] = [
|
||||
'.rst',
|
||||
'.md',
|
||||
'.mdx',
|
||||
'.pdf',
|
||||
'.txt',
|
||||
'.docx',
|
||||
'.csv',
|
||||
'.epub',
|
||||
'.html',
|
||||
'.xhtml',
|
||||
'.json',
|
||||
'.xlsx',
|
||||
'.pptx',
|
||||
'.adoc',
|
||||
'.asciidoc',
|
||||
'.png',
|
||||
'.jpg',
|
||||
'.jpeg',
|
||||
'.tiff',
|
||||
'.tif',
|
||||
'.bmp',
|
||||
'.webp',
|
||||
'.vtt',
|
||||
'.xml',
|
||||
'.wav',
|
||||
'.mp3',
|
||||
'.m4a',
|
||||
'.ogg',
|
||||
'.webm',
|
||||
];
|
||||
|
||||
/** Picker filter for the Attach button. A hint only — pickers may ignore it. */
|
||||
export const ATTACHMENT_FILE_ACCEPT_ATTR =
|
||||
ATTACHMENT_PARSER_EXTENSIONS.join(',');
|
||||
|
||||
/** Lower-cased last extension including the dot, or '' (dotfiles have none). */
|
||||
export function getFileExtension(name: string): string {
|
||||
const base = name.split(/[\\/]/).pop() ?? '';
|
||||
const dot = base.lastIndexOf('.');
|
||||
if (dot <= 0) return '';
|
||||
return base.slice(dot).toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the backend has a parser for this suffix. False is not a refusal:
|
||||
* the file then has to read as text. Never decided by the mime type the
|
||||
* picker reports — mobile pickers ignore `accept` and lie about types.
|
||||
*/
|
||||
export function hasAttachmentParser(file: { name: string }): boolean {
|
||||
const ext = getFileExtension(file.name);
|
||||
return ext !== '' && ATTACHMENT_PARSER_EXTENSIONS.includes(ext);
|
||||
}
|
||||
|
||||
// Mirrors application/upload_limits.py: enough of the head to recognise a
|
||||
// container header, and a tolerance that keeps real text (UTF-8 accents, an
|
||||
// ANSI-coloured log) in while a random binary stays out.
|
||||
const TEXT_SNIFF_BYTES = 8192;
|
||||
const MAX_NONTEXT_RATIO = 0.1;
|
||||
// Control bytes that occur in ordinary text: tab, LF, VT, FF, CR, ESC.
|
||||
const TEXT_CONTROL_BYTES = new Set([0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x1b]);
|
||||
|
||||
/** Whether a leading byte sample reads as text rather than binary. */
|
||||
export function looksLikeText(sample: Uint8Array): boolean {
|
||||
if (sample.length === 0) return true;
|
||||
let nontext = 0;
|
||||
for (const byte of sample) {
|
||||
if (byte === 0x00) return false;
|
||||
if ((byte < 0x20 && !TEXT_CONTROL_BYTES.has(byte)) || byte === 0x7f)
|
||||
nontext += 1;
|
||||
}
|
||||
return nontext / sample.length <= MAX_NONTEXT_RATIO;
|
||||
}
|
||||
|
||||
async function isSupportedAttachmentFile(file: File): Promise<boolean> {
|
||||
if (hasAttachmentParser(file)) return true;
|
||||
try {
|
||||
const head = await file.slice(0, TEXT_SNIFF_BYTES).arrayBuffer();
|
||||
return looksLikeText(new Uint8Array(head));
|
||||
} catch {
|
||||
// Can't read it here — let the upload run and the server decide.
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Split a selection into what the backend can read and what it will refuse,
|
||||
* preserving order. Files with a parser pass on their suffix; the rest are
|
||||
* sniffed, so source, config and log files go through and a video does not.
|
||||
*/
|
||||
export async function partitionAttachmentFiles(files: File[]): Promise<{
|
||||
supported: File[];
|
||||
unsupported: File[];
|
||||
}> {
|
||||
const verdicts = await Promise.all(files.map(isSupportedAttachmentFile));
|
||||
const supported: File[] = [];
|
||||
const unsupported: File[] = [];
|
||||
files.forEach((file, index) => {
|
||||
(verdicts[index] ? supported : unsupported).push(file);
|
||||
});
|
||||
return { supported, unsupported };
|
||||
}
|
||||
|
||||
/** The ``message`` of a JSON error body, if the server sent one. */
|
||||
export function parseUploadErrorMessage(body: string): string | undefined {
|
||||
if (!body) return undefined;
|
||||
try {
|
||||
const parsed = JSON.parse(body) as { message?: unknown };
|
||||
return typeof parsed?.message === 'string' && parsed.message
|
||||
? parsed.message
|
||||
: undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
@@ -1101,7 +1101,8 @@
|
||||
"waitingToSend_one": "Will send when {{count}} file finishes processing…",
|
||||
"waitingToSend_other": "Will send when {{count}} files finish processing…",
|
||||
"cancelQueuedSend": "Cancel",
|
||||
"sendBlockedByFailed": "{{names}} couldn't be processed — remove to send"
|
||||
"sendBlockedByFailed": "{{names}} couldn't be processed — remove to send",
|
||||
"unsupportedType": "Not a supported file type ({{extension}})"
|
||||
},
|
||||
"retry": "Retry",
|
||||
"reasoning": "Reasoning",
|
||||
|
||||
@@ -46,6 +46,8 @@ export interface Attachment {
|
||||
*/
|
||||
attachmentId?: string;
|
||||
token_count?: number;
|
||||
/** Why a ``failed`` attachment failed, when known (server or client gate). */
|
||||
errorMessage?: string;
|
||||
}
|
||||
|
||||
export type UploadTaskStatus =
|
||||
|
||||
@@ -2386,3 +2386,97 @@ class TestRequireLiveSttRedisUnavailable:
|
||||
result = _require_live_stt_redis()
|
||||
assert hasattr(result, "status_code")
|
||||
assert result.status_code == 503
|
||||
|
||||
|
||||
MP4_BYTES = b"\x00\x00\x00\x18ftypisom\x00\x00\x02\x00isomiso2avc1mp41\x00\x00\x00\x08free"
|
||||
|
||||
|
||||
class TestStoreAttachmentTypeGate:
|
||||
"""Unparseable uploads are refused before anything is stored or queued."""
|
||||
|
||||
@patch("application.api.user.tasks.store_attachment.delay")
|
||||
def test_rejects_unsupported_file_type_before_storage(
|
||||
self, mock_store_attachment, flask_app
|
||||
):
|
||||
from application.api.user.attachments.routes import StoreAttachment
|
||||
|
||||
app = Flask(__name__)
|
||||
mock_storage = MagicMock()
|
||||
with patch("application.api.user.base.storage", mock_storage), app.test_request_context(
|
||||
"/api/store_attachment",
|
||||
method="POST",
|
||||
data={"file": (io.BytesIO(MP4_BYTES), "clip.mp4")},
|
||||
content_type="multipart/form-data",
|
||||
):
|
||||
request.decoded_token = {"sub": "test_user"}
|
||||
response = StoreAttachment().post()
|
||||
|
||||
assert _get_response_status(response) == 400
|
||||
body = _get_response_json(response)
|
||||
assert body["success"] is False
|
||||
assert body["message"] == "Unsupported file type: .mp4"
|
||||
assert body["errors"] == [
|
||||
{"upload_index": 0, "filename": "clip.mp4", "error": "Unsupported file type: .mp4"}
|
||||
]
|
||||
mock_storage.save_file.assert_not_called()
|
||||
mock_store_attachment.assert_not_called()
|
||||
|
||||
@patch("application.api.user.tasks.store_attachment.delay")
|
||||
def test_accepts_a_text_file_with_no_dedicated_parser(
|
||||
self, mock_store_attachment, flask_app
|
||||
):
|
||||
"""A .py or .log is read by the plain-text fallthrough and must stay allowed."""
|
||||
from application.api.user.attachments.routes import StoreAttachment
|
||||
|
||||
app = Flask(__name__)
|
||||
mock_storage = MagicMock()
|
||||
mock_storage.save_file.return_value = {"storage_type": "local"}
|
||||
mock_store_attachment.return_value = SimpleNamespace(id="task-py")
|
||||
|
||||
with patch("application.api.user.base.storage", mock_storage), app.test_request_context(
|
||||
"/api/store_attachment",
|
||||
method="POST",
|
||||
data={"file": (io.BytesIO(b"def main():\n return 1\n"), "main.py")},
|
||||
content_type="multipart/form-data",
|
||||
):
|
||||
request.decoded_token = {"sub": "test_user"}
|
||||
response = StoreAttachment().post()
|
||||
|
||||
assert _get_response_status(response) == 200
|
||||
assert _get_response_json(response)["task_id"] == "task-py"
|
||||
assert mock_store_attachment.call_count == 1
|
||||
|
||||
@patch("application.api.user.tasks.store_attachment.delay")
|
||||
def test_batch_skips_unsupported_file_and_keeps_the_rest(
|
||||
self, mock_store_attachment, flask_app
|
||||
):
|
||||
from application.api.user.attachments.routes import StoreAttachment
|
||||
|
||||
app = Flask(__name__)
|
||||
mock_storage = MagicMock()
|
||||
mock_storage.save_file.return_value = {"storage_type": "local"}
|
||||
mock_store_attachment.return_value = SimpleNamespace(id="task-notes")
|
||||
|
||||
with patch("application.api.user.base.storage", mock_storage), app.test_request_context(
|
||||
"/api/store_attachment",
|
||||
method="POST",
|
||||
data={
|
||||
"file": [
|
||||
(io.BytesIO(MP4_BYTES), "clip.mp4"),
|
||||
(io.BytesIO(b"hello"), "notes.txt"),
|
||||
]
|
||||
},
|
||||
content_type="multipart/form-data",
|
||||
):
|
||||
request.decoded_token = {"sub": "test_user"}
|
||||
response = StoreAttachment().post()
|
||||
|
||||
assert _get_response_status(response) == 200
|
||||
body = _get_response_json(response)
|
||||
assert [task["upload_index"] for task in body["tasks"]] == [1]
|
||||
assert body["tasks"][0]["filename"] == "notes.txt"
|
||||
assert body["errors"] == [
|
||||
{"upload_index": 0, "filename": "clip.mp4", "error": "Unsupported file type: .mp4"}
|
||||
]
|
||||
assert mock_storage.save_file.call_count == 1
|
||||
assert mock_store_attachment.call_count == 1
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Tests for the shared file-extension constants."""
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from application.parser.file.bulk import get_default_file_extractor
|
||||
from application.parser.file.constants import (
|
||||
attachment_extension,
|
||||
has_attachment_parser,
|
||||
SUPPORTED_ATTACHMENT_EXTENSIONS,
|
||||
SUPPORTED_SOURCE_EXTENSIONS,
|
||||
)
|
||||
|
||||
FRONTEND_CONSTANTS = (
|
||||
Path(__file__).resolve().parents[3] / "frontend/src/constants/fileUpload.ts"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_every_parser_backed_suffix_is_an_allowed_attachment():
|
||||
"""The allow-list is a hand-kept literal; this is what keeps it honest.
|
||||
|
||||
``SUPPORTED_ATTACHMENT_EXTENSIONS`` cannot import the extractor (that
|
||||
would pull docling into the API process), so a parser added to
|
||||
``get_default_file_extractor`` would otherwise stay refused as an
|
||||
attachment — which is how .webp, .tiff, .vtt and .xml were first missed.
|
||||
"""
|
||||
parser_suffixes = set(get_default_file_extractor())
|
||||
|
||||
assert parser_suffixes <= set(SUPPORTED_ATTACHMENT_EXTENSIONS), (
|
||||
"parsers exist for suffixes the attachment gate refuses: "
|
||||
f"{sorted(parser_suffixes - set(SUPPORTED_ATTACHMENT_EXTENSIONS))}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_frontend_mirrors_the_attachment_allow_list():
|
||||
"""The composer gates uploads client-side; a divergent list refuses valid files.
|
||||
|
||||
``ATTACHMENT_PARSER_EXTENSIONS`` in the frontend is a hand-kept copy of
|
||||
``SUPPORTED_ATTACHMENT_EXTENSIONS``. Nothing but this test connects them,
|
||||
and .mdx, .xhtml and .adoc were blocked in the UI while the API accepted
|
||||
them before it existed.
|
||||
"""
|
||||
if not FRONTEND_CONSTANTS.exists():
|
||||
pytest.skip("frontend sources not present in this checkout")
|
||||
|
||||
source = FRONTEND_CONSTANTS.read_text(encoding="utf-8")
|
||||
match = re.search(
|
||||
r"ATTACHMENT_PARSER_EXTENSIONS:\s*readonly string\[\]\s*=\s*\[(.*?)\]",
|
||||
source,
|
||||
re.DOTALL,
|
||||
)
|
||||
assert match, "ATTACHMENT_PARSER_EXTENSIONS not found in fileUpload.ts"
|
||||
frontend_extensions = set(re.findall(r"'(\.[^']+)'", match.group(1)))
|
||||
|
||||
assert frontend_extensions == set(SUPPORTED_ATTACHMENT_EXTENSIONS), (
|
||||
"frontend and backend attachment lists disagree — "
|
||||
f"backend only: {sorted(set(SUPPORTED_ATTACHMENT_EXTENSIONS) - frontend_extensions)}, "
|
||||
f"frontend only: {sorted(frontend_extensions - set(SUPPORTED_ATTACHMENT_EXTENSIONS))}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_attachments_accept_everything_source_ingestion_does():
|
||||
assert set(SUPPORTED_SOURCE_EXTENSIONS) <= set(SUPPORTED_ATTACHMENT_EXTENSIONS)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@pytest.mark.parametrize(
|
||||
("filename", "expected"),
|
||||
[
|
||||
("Photo.JPG", ".jpg"),
|
||||
("/tmp/dir.d/report.PDF", ".pdf"),
|
||||
("archive.tar.gz", ".gz"),
|
||||
("Dockerfile", ""),
|
||||
(".env", ""),
|
||||
("trailing.", "."),
|
||||
(None, ""),
|
||||
],
|
||||
)
|
||||
def test_attachment_extension(filename, expected):
|
||||
assert attachment_extension(filename) == expected
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_has_attachment_parser_is_case_insensitive_and_excludes_zip():
|
||||
assert has_attachment_parser("Report.PDF")
|
||||
assert has_attachment_parser("scan.WebP")
|
||||
# No parser: admitted (or not) on content instead, never on the suffix.
|
||||
assert not has_attachment_parser("archive.zip")
|
||||
assert not has_attachment_parser("main.py")
|
||||
assert not has_attachment_parser("clip.mp4")
|
||||
@@ -37,3 +37,126 @@ def test_limited_text_read_decodes_incrementally_and_rejects_overflow():
|
||||
|
||||
with pytest.raises(UploadTooLargeError):
|
||||
read_text_upload_limited(_ShortReadStream(b"12345"), max_bytes=4)
|
||||
|
||||
|
||||
# --- attachment type gate -------------------------------------------------
|
||||
#
|
||||
# ``SimpleDirectoryReader`` falls through to a plain-text ``open()`` for any
|
||||
# suffix without a parser. That is what a .py or a .log attachment relies on,
|
||||
# and it is also how a phone-uploaded video used to be "parsed" into
|
||||
# megabytes of binary garbage, truncated, and stored with
|
||||
# ``extraction.status == "ok"``. So a suffix with no parser is admitted on
|
||||
# content: text in, binary out.
|
||||
|
||||
MP4_HEADER = b"\x00\x00\x00\x18ftypisom\x00\x00\x02\x00isomiso2avc1mp41"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("filename", "content"),
|
||||
[
|
||||
("clip.mp4", MP4_HEADER),
|
||||
("movie.MOV", MP4_HEADER),
|
||||
("archive.zip", b"PK\x03\x04\x14\x00\x00\x00\x08\x00" + bytes(range(32))),
|
||||
("binary", bytes(range(32)) * 8),
|
||||
("trailing.", b"\x00\x01\x02\x03"),
|
||||
("x.tar.gz", b"\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03"),
|
||||
],
|
||||
)
|
||||
def test_enforce_parseable_attachment_rejects_binary_without_a_parser(
|
||||
filename, content, tmp_path
|
||||
):
|
||||
from application.upload_limits import (
|
||||
enforce_parseable_attachment,
|
||||
UnsupportedUploadTypeError,
|
||||
unsupported_upload_message,
|
||||
)
|
||||
|
||||
path = tmp_path / "staged.bin"
|
||||
path.write_bytes(content)
|
||||
|
||||
with pytest.raises(UnsupportedUploadTypeError) as excinfo:
|
||||
enforce_parseable_attachment(path, filename)
|
||||
assert str(excinfo.value) == unsupported_upload_message(filename)
|
||||
assert str(excinfo.value).startswith("Unsupported file type")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"filename",
|
||||
[
|
||||
"notes.txt",
|
||||
"Report.PDF",
|
||||
"photo.JPG",
|
||||
"slides.pptx",
|
||||
"voice.ogg",
|
||||
"page.xhtml",
|
||||
"doc.adoc",
|
||||
"scan.webp",
|
||||
"fax.tiff",
|
||||
"subs.vtt",
|
||||
"feed.xml",
|
||||
],
|
||||
)
|
||||
def test_enforce_parseable_attachment_accepts_parser_backed_types(filename, tmp_path):
|
||||
"""A parser-backed suffix is admitted on its name — a PDF is binary and parses fine."""
|
||||
from application.upload_limits import enforce_parseable_attachment
|
||||
|
||||
path = tmp_path / "staged.bin"
|
||||
path.write_bytes(MP4_HEADER)
|
||||
|
||||
enforce_parseable_attachment(path, filename)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"filename",
|
||||
["main.py", "server.log", "config.yaml", "query.sql", "Dockerfile", "notes.unknown"],
|
||||
)
|
||||
def test_enforce_parseable_attachment_accepts_text_without_a_parser(filename, tmp_path):
|
||||
"""The plain-text fallthrough reads these correctly, so they must stay allowed."""
|
||||
from application.upload_limits import enforce_parseable_attachment
|
||||
|
||||
path = tmp_path / "staged.txt"
|
||||
path.write_text("def main():\n\treturn 'café — ok'\n", encoding="utf-8")
|
||||
|
||||
enforce_parseable_attachment(path, filename)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("sample", "expected"),
|
||||
[
|
||||
(b"", True),
|
||||
(b"plain text\n", True),
|
||||
("café — em dash\n".encode(), True),
|
||||
(b"\x1b[31mred log line\x1b[0m\n", True),
|
||||
(b"text\x00with nul", False),
|
||||
(bytes(range(32)) * 4, False),
|
||||
(b"\x7f\x7f\x7f\x7f" + b"a" * 16, False),
|
||||
],
|
||||
)
|
||||
def test_looks_like_text(sample, expected):
|
||||
from application.upload_limits import looks_like_text
|
||||
|
||||
assert looks_like_text(sample) is expected
|
||||
|
||||
|
||||
def test_file_looks_like_text_only_samples_the_head(tmp_path):
|
||||
"""Binary past the sampled head is the parser's problem, not the gate's."""
|
||||
from application.upload_limits import file_looks_like_text
|
||||
|
||||
path = tmp_path / "staged.log"
|
||||
path.write_bytes(b"a" * 9000 + b"\x00" * 100)
|
||||
|
||||
assert file_looks_like_text(path) is True
|
||||
|
||||
|
||||
def test_file_looks_like_text_allows_an_unreadable_file(tmp_path):
|
||||
from application.upload_limits import file_looks_like_text
|
||||
|
||||
assert file_looks_like_text(tmp_path / "missing.txt") is True
|
||||
|
||||
|
||||
def test_unsupported_upload_message_names_the_extension():
|
||||
from application.upload_limits import unsupported_upload_message
|
||||
|
||||
assert unsupported_upload_message("clip.mp4") == "Unsupported file type: .mp4"
|
||||
assert unsupported_upload_message("Clip.MP4") == "Unsupported file type: .mp4"
|
||||
assert unsupported_upload_message("binary") == "Unsupported file type: (no extension)"
|
||||
@@ -326,3 +326,86 @@ class TestAttachmentZipBombGuard:
|
||||
|
||||
# BadZipFile → return quietly; the format parser surfaces a clean error.
|
||||
worker._reject_attachment_zip_bomb(str(path))
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestAttachmentTypeGuard:
|
||||
"""A binary with no parser must fail rather than be read as text.
|
||||
|
||||
``SimpleDirectoryReader`` falls through to a plain-text ``open()`` for a
|
||||
suffix it has no parser for, which is how a phone-uploaded ``.mp4`` was
|
||||
once stored as 100k tokens of binary with ``extraction.status == "ok"``.
|
||||
The same fallthrough is what makes a ``.py`` or ``.log`` attachment work,
|
||||
so the guard admits those on content.
|
||||
"""
|
||||
|
||||
def test_binary_without_a_parser_fails_instead_of_being_read_as_text(
|
||||
self, pg_conn, patch_worker_db, task_self, monkeypatch, tmp_path
|
||||
):
|
||||
from application import worker
|
||||
|
||||
local_path = tmp_path / "clip.mp4"
|
||||
local_path.write_bytes(b"\x00\x00\x00\x18ftypisom\x00\x00\x02\x00isomavc1")
|
||||
|
||||
events = []
|
||||
fake_storage = MagicMock(name="storage")
|
||||
fake_storage.process_file.side_effect = lambda path, callback: callback(
|
||||
str(local_path)
|
||||
)
|
||||
monkeypatch.setattr(worker.StorageCreator, "get_storage", lambda: fake_storage)
|
||||
monkeypatch.setattr(
|
||||
worker,
|
||||
"get_default_file_extractor",
|
||||
lambda ocr_enabled=False, pdf_text_fast_path=False: {},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
worker,
|
||||
"publish_user_event",
|
||||
lambda user, name, payload, **kwargs: events.append((name, payload)),
|
||||
)
|
||||
|
||||
file_info = {
|
||||
"filename": "clip.mp4",
|
||||
"attachment_id": "507f1f77bcf86cd799439012",
|
||||
"path": "uploads/user1/attachments/clip.mp4",
|
||||
"metadata": {"source": "chat"},
|
||||
}
|
||||
|
||||
with pytest.raises(
|
||||
worker.AttachmentRejectedError, match=r"Unsupported file type: \.mp4"
|
||||
):
|
||||
worker.attachment_worker(task_self, file_info, "user1")
|
||||
|
||||
failed = [payload for name, payload in events if name == "attachment.failed"]
|
||||
assert failed and failed[0]["error"] == "Unsupported file type: .mp4"
|
||||
|
||||
def test_text_without_a_parser_is_parsed(
|
||||
self, pg_conn, patch_worker_db, task_self, monkeypatch, tmp_path
|
||||
):
|
||||
from application import worker
|
||||
|
||||
local_path = tmp_path / "server.log"
|
||||
local_path.write_text("2026-09-02 ERROR boom\n", encoding="utf-8")
|
||||
|
||||
fake_storage = MagicMock(name="storage")
|
||||
fake_storage.process_file.side_effect = lambda path, callback: callback(
|
||||
str(local_path)
|
||||
)
|
||||
monkeypatch.setattr(worker.StorageCreator, "get_storage", lambda: fake_storage)
|
||||
monkeypatch.setattr(
|
||||
worker,
|
||||
"get_default_file_extractor",
|
||||
lambda ocr_enabled=False, pdf_text_fast_path=False: {},
|
||||
)
|
||||
|
||||
file_info = {
|
||||
"filename": "server.log",
|
||||
"attachment_id": "507f1f77bcf86cd799439013",
|
||||
"path": "uploads/user1/attachments/server.log",
|
||||
"metadata": {"source": "chat"},
|
||||
}
|
||||
|
||||
result = worker.attachment_worker(task_self, file_info, "user1")
|
||||
|
||||
assert result["filename"] == "server.log"
|
||||
assert result["token_count"] > 0
|
||||
Reference in new issue
Block a user