Merge branch 'main' into anydoc-support

Conflicts, and how each was taken:

- application/core/settings.py — ours. The renamed OCR_ENABLED /
  OCR_ATTACHMENTS_ENABLED / OCR_MIN_CHARS_PER_PAGE accept main's
  DOCLING_OCR_* spellings as AliasChoices, so nothing is dropped.
- application/Dockerfile — both. Main's install layers plus the
  INSTALL_DOCLING build arg.
- application/parser/file/constants.py — both imports.
- deployment/docker-compose.yaml — both. The INSTALL_DOCLING /
  INSTALL_TESSERACT build args on backend and worker, and main's
  -Q docsgpt,parsing,embeddings, which query embedding needs.
- tests/conftest.py — theirs. Both sides fixed the same pytest-postgresql
  9.0.0 autocommit= breakage; main's spelling is the one already on main.
- application/requirements.txt — the comments claimed different reasons
  torch is in core. Main's is the true one now: it removed
  sentence-transformers, so docling is torch's only remaining consumer.

Two things the merge broke without conflicting:

- onnxruntime. This branch moved it out of core into the docling extra;
  main meanwhile made it the runtime local embeddings execute on
  (fastembed). Git took the deletion, leaving fastembed with no pinned
  runtime in a repo that pins everything. Restored to core, and no longer
  pinned twice from the extra.
- The frontend copy of ATTACHMENT_PARSER_EXTENSIONS. The backend list is
  derived and picked up the anydoc suffixes; the hand-kept frontend mirror
  did not, so the composer would refuse files the API accepts.
  tests/parser/file/test_constants.py is what caught it.

ruff, pytest (9897 passed), frontend build and docs build all pass. The
image build is unverified: no Docker daemon on this machine.
This commit is contained in:
Alex committed 2026-09-04 16:42:48 +01:00
commit 3947c66cda
97 files changed
+8661 -1049

No files matched your search

+65 -10
View File
@@ -48,7 +48,10 @@ import { useArmedSend } from './message-input/armedSend';
import { handleAbort } from '../conversation/conversationSlice';
import {
AUDIO_FILE_ACCEPT_ATTR,
FILE_UPLOAD_ACCEPT,
getFileExtension,
parseUploadErrorMessage,
parseUploadErrorsByIndex,
partitionAttachmentFiles,
} from '../constants/fileUpload';
import { UserToolType } from '../settings/types';
import { isChatToolVisible } from '../utils/toolUtils';
@@ -497,8 +500,31 @@ export default function MessageInput({
);
const uploadFiles = useCallback(
(files: File[]) => {
if (!files || files.length === 0) return;
async (incomingFiles: File[]) => {
if (!incomingFiles || incomingFiles.length === 0) return;
// Run the server's own rule here, not just the input's `accept`:
// mobile pickers ignore `accept`, and a file the server will refuse
// should say so before it costs an upload. Surface the refusal as a
// failed chip so the user sees why instead of a silent drop.
const { supported, unsupported } =
await partitionAttachmentFiles(incomingFiles);
unsupported.forEach((file) => {
dispatch(
addAttachment({
id: generateId(),
fileName: file.name,
progress: 0,
status: 'failed' as const,
taskId: '',
errorMessage: t('conversation.attachments.unsupportedType', {
extension: getFileExtension(file.name) || '?',
}),
}),
);
});
if (supported.length === 0) return;
const files = supported;
const apiHost = import.meta.env.VITE_API_HOST;
@@ -579,9 +605,14 @@ export default function MessageInput({
tasksByIndex.set(uploadIndex, task);
});
const errorsByIndex = new Map<number, string | undefined>();
errors.forEach((errorItem) => {
if (typeof errorItem.upload_index === 'number') {
failedIndices.add(errorItem.upload_index);
errorsByIndex.set(
errorItem.upload_index,
errorItem.error,
);
}
});
@@ -616,7 +647,10 @@ export default function MessageInput({
dispatch(
updateAttachment({
id: uiId,
updates: { status: 'failed' },
updates: {
status: 'failed',
errorMessage: errorsByIndex.get(index),
},
}),
);
return;
@@ -748,11 +782,20 @@ export default function MessageInput({
}
} else {
console.error('Upload failed', status, xhr.responseText);
Object.values(indexToUiId).forEach((id) =>
// Each file gets its own reason where the server sent one; the
// top-level message is the fallback, not the answer for all of
// them — a batch can fail two files for two different reasons.
const fallbackMessage = parseUploadErrorMessage(xhr.responseText);
const errorsByIndex = parseUploadErrorsByIndex(xhr.responseText);
Object.entries(indexToUiId).forEach(([index, id]) =>
dispatch(
updateAttachment({
id,
updates: { status: 'failed' },
updates: {
status: 'failed',
errorMessage:
errorsByIndex.get(Number(index)) ?? fallbackMessage,
},
}),
),
);
@@ -871,7 +914,10 @@ export default function MessageInput({
dispatch(
updateAttachment({
id: uniqueId,
updates: { status: 'failed' },
updates: {
status: 'failed',
errorMessage: parseUploadErrorMessage(xhr.responseText),
},
}),
);
}
@@ -891,7 +937,7 @@ export default function MessageInput({
xhr.send(formData);
});
},
[dispatch, token, trackAttachment],
[dispatch, t, token, trackAttachment],
);
const handleFileAttachment = (e: React.ChangeEvent<HTMLInputElement>) => {
@@ -928,7 +974,10 @@ export default function MessageInput({
setHandleDragActive(false);
},
maxSize: 25000000,
accept: FILE_UPLOAD_ACCEPT,
// No `accept`: react-dropzone would drop a rejected file on the floor
// with no feedback, and its mime matching disagrees with the server for
// text files that have no parser (.py, .log). uploadFiles applies the
// server's rule and reports what it refuses.
});
const handleInput = useCallback(() => {
@@ -1594,7 +1643,13 @@ export default function MessageInput({
onChange={handleVoiceFileAttachment}
/>
<div className="border-border bg-card relative flex w-full flex-col rounded-3xl border dark:bg-transparent">
{/* translate="no": keep Chrome's page translator out of the composer —
it rewrites text nodes into <font> wrappers and React loses the
controls (dead Attach button, see AttachFileButton). */}
<div
translate="no"
className="border-border bg-card relative flex w-full flex-col rounded-3xl border dark:bg-transparent"
>
<AttachmentChipList
attachments={attachments}
draggingId={draggingId}
@@ -1,7 +1,7 @@
import { useTranslation } from 'react-i18next';
import ClipIcon from '../../assets/clip.svg';
import { FILE_UPLOAD_ACCEPT_ATTR } from '../../constants/fileUpload';
import { ATTACHMENT_FILE_ACCEPT_ATTR } from '../../constants/fileUpload';
type AttachFileButtonProps = {
onChange: (e: React.ChangeEvent<HTMLInputElement>) => void;
@@ -11,7 +11,14 @@ export default function AttachFileButton({ onChange }: AttachFileButtonProps) {
const { t } = useTranslation();
return (
<label className="xs:px-3 xs:py-1.5 dark:border-border border-border hover:bg-muted dark:hover:bg-muted flex cursor-pointer items-center rounded-full border px-2 py-1 transition-colors">
// translate="no": Chrome's page translator rewrites text nodes inside
// this label into <font> wrappers and React then loses the control — a
// user with auto-translate on rage-clicked a dead Attach button until
// they reloaded. Keep the composer's controls out of the translator.
<label
translate="no"
className="xs:px-3 xs:py-1.5 dark:border-border border-border hover:bg-muted dark:hover:bg-muted flex cursor-pointer items-center rounded-full border px-2 py-1 transition-colors"
>
<img
src={ClipIcon}
alt="Attach"
@@ -24,7 +31,7 @@ export default function AttachFileButton({ onChange }: AttachFileButtonProps) {
type="file"
className="hidden"
multiple
accept={FILE_UPLOAD_ACCEPT_ATTR}
accept={ATTACHMENT_FILE_ACCEPT_ATTR}
onChange={onChange}
/>
</label>
@@ -25,94 +25,122 @@ export default function AttachmentChipList({
}: AttachmentChipListProps) {
const { t } = useTranslation();
// A tooltip is the one place a touch user can never look, and this list is
// where a phone picker's unsupported file lands. Show the reason inline,
// as soon as it is known, rather than only once a send is attempted.
const failures = attachments.filter(
(attachment) => attachment.status === 'failed' && attachment.errorMessage,
);
return (
<div className="flex flex-wrap gap-1.5 px-2 py-2 sm:gap-2 sm:px-3">
{attachments.map((attachment) => {
return (
<div
key={attachment.id}
draggable={true}
onDragStart={(e) => onDragStart(e, attachment.id)}
onDragOver={onDragOver}
onDrop={(e) => onDropOn(e, attachment.id)}
className={`group dark:text-foreground bg-muted text-muted-foreground dark:bg-accent relative flex items-center rounded-xl px-2 py-1 text-xs sm:px-3 sm:py-1.5 sm:text-sm ${
attachment.status !== 'completed' ? 'opacity-70' : 'opacity-100'
} ${
draggingId === attachment.id
? 'ring-dashed opacity-60 ring-2 ring-purple-200'
: ''
}`}
title={attachment.fileName}
>
<div className="bg-primary mr-2 flex h-8 w-8 items-center justify-center rounded-md p-1">
{attachment.status === 'completed' && (
<img
src={DocumentationDark}
alt="Attachment"
className="h-[15px] w-[15px] object-fill"
/>
)}
{attachment.status === 'failed' && (
<img
src={AlertIcon}
alt="Failed"
className="h-[15px] w-[15px] object-fill"
/>
)}
{(attachment.status === 'uploading' ||
attachment.status === 'processing') && (
<div className="flex h-[15px] w-[15px] items-center justify-center">
<svg className="h-[15px] w-[15px]" viewBox="0 0 24 24">
<circle
className="opacity-0"
cx="12"
cy="12"
r="10"
stroke="transparent"
strokeWidth="4"
fill="none"
/>
<circle
className="text-[#ECECF1]"
cx="12"
cy="12"
r="10"
stroke="currentColor"
strokeWidth="4"
fill="none"
strokeDasharray="62.83"
strokeDashoffset={62.83 * (1 - attachment.progress / 100)}
transform="rotate(-90 12 12)"
/>
</svg>
</div>
)}
</div>
<span className="max-w-[120px] truncate font-medium sm:max-w-[150px]">
{attachment.fileName}
</span>
<Button
type="button"
variant="ghost"
size="icon-sm"
className="ml-1.5 h-auto w-auto rounded-full p-1"
onClick={() => {
onRemove(attachment.id);
}}
aria-label={t('conversation.attachments.remove')}
<>
<div className="flex flex-wrap gap-1.5 px-2 py-2 sm:gap-2 sm:px-3">
{attachments.map((attachment) => {
return (
<div
key={attachment.id}
draggable={true}
onDragStart={(e) => onDragStart(e, attachment.id)}
onDragOver={onDragOver}
onDrop={(e) => onDropOn(e, attachment.id)}
className={`group dark:text-foreground bg-muted text-muted-foreground dark:bg-accent relative flex items-center rounded-xl px-2 py-1 text-xs sm:px-3 sm:py-1.5 sm:text-sm ${
attachment.status !== 'completed' ? 'opacity-70' : 'opacity-100'
} ${
draggingId === attachment.id
? 'ring-dashed opacity-60 ring-2 ring-purple-200'
: ''
}`}
title={
attachment.status === 'failed' && attachment.errorMessage
? `${attachment.fileName}: ${attachment.errorMessage}`
: attachment.fileName
}
>
<X
<div className="bg-primary mr-2 flex h-8 w-8 items-center justify-center rounded-md p-1">
{attachment.status === 'completed' && (
<img
src={DocumentationDark}
alt="Attachment"
className="h-[15px] w-[15px] object-fill"
/>
)}
{attachment.status === 'failed' && (
<img
src={AlertIcon}
alt="Failed"
className="h-[15px] w-[15px] object-fill"
/>
)}
{(attachment.status === 'uploading' ||
attachment.status === 'processing') && (
<div className="flex h-[15px] w-[15px] items-center justify-center">
<svg className="h-[15px] w-[15px]" viewBox="0 0 24 24">
<circle
className="opacity-0"
cx="12"
cy="12"
r="10"
stroke="transparent"
strokeWidth="4"
fill="none"
/>
<circle
className="text-[#ECECF1]"
cx="12"
cy="12"
r="10"
stroke="currentColor"
strokeWidth="4"
fill="none"
strokeDasharray="62.83"
strokeDashoffset={
62.83 * (1 - attachment.progress / 100)
}
transform="rotate(-90 12 12)"
/>
</svg>
</div>
)}
</div>
<span className="max-w-[120px] truncate font-medium sm:max-w-[150px]">
{attachment.fileName}
</span>
<Button
type="button"
variant="ghost"
size="icon-sm"
className="ml-1.5 h-auto w-auto rounded-full p-1"
onClick={() => {
onRemove(attachment.id);
}}
aria-label={t('conversation.attachments.remove')}
className="h-2.5 w-2.5"
/>
</Button>
</div>
);
})}
</div>
>
<X
aria-label={t('conversation.attachments.remove')}
className="h-2.5 w-2.5"
/>
</Button>
</div>
);
})}
</div>
{failures.length > 0 && (
<div
className="flex flex-col gap-0.5 px-2 pb-1 text-xs text-[#B42318] sm:px-3"
role="alert"
>
{failures.map((attachment) => (
<span key={attachment.id}>
{attachment.fileName}: {attachment.errorMessage}
</span>
))}
</div>
)}
</>
);
}
+240
View File
@@ -0,0 +1,240 @@
import { describe, expect, it } from 'vitest';
import {
ATTACHMENT_FILE_ACCEPT_ATTR,
ATTACHMENT_PARSER_EXTENSIONS,
getFileExtension,
hasAttachmentParser,
looksLikeText,
parseUploadErrorMessage,
parseUploadErrorsByIndex,
partitionAttachmentFiles,
} from './fileUpload';
const file = (name: string, body: BlobPart = 'x', type = '') =>
new File([body], name, { type });
const bytes = (...values: number[]) => new Uint8Array(values);
const MP4_HEADER = bytes(
0x00,
0x00,
0x00,
0x18,
0x66,
0x74,
0x79,
0x70,
0x69,
0x73,
0x6f,
0x6d,
);
describe('attachment type gate', () => {
it('extracts a lower-cased extension', () => {
expect(getFileExtension('Photo.JPG')).toBe('.jpg');
expect(getFileExtension('a.tar.gz')).toBe('.gz');
expect(getFileExtension('noext')).toBe('');
expect(getFileExtension('.env')).toBe('');
});
it('recognises parser-backed suffixes by name, case-insensitively', () => {
expect(hasAttachmentParser(file('Report.PDF'))).toBe(true);
expect(hasAttachmentParser(file('scan.WebP'))).toBe(true);
expect(hasAttachmentParser(file('subs.vtt'))).toBe(true);
// No parser — these are judged on content, not on the suffix.
expect(hasAttachmentParser(file('main.py'))).toBe(false);
expect(hasAttachmentParser(file('archive.zip'))).toBe(false);
expect(hasAttachmentParser(file('clip.mp4'))).toBe(false);
// .txt is the plain-text fallthrough itself, not a parser.
expect(hasAttachmentParser(file('notes.txt'))).toBe(false);
});
it('mirrors the backend list: no zip, no video, images and markup included', () => {
expect(ATTACHMENT_PARSER_EXTENSIONS).toContain('.pdf');
// Parser-backed suffixes the first cut of this list missed.
for (const ext of [
'.webp',
'.tiff',
'.tif',
'.bmp',
'.vtt',
'.xml',
'.mdx',
]) {
expect(ATTACHMENT_PARSER_EXTENSIONS).toContain(ext);
}
expect(ATTACHMENT_PARSER_EXTENSIONS).not.toContain('.zip');
expect(ATTACHMENT_PARSER_EXTENSIONS).not.toContain('.mp4');
expect(ATTACHMENT_PARSER_EXTENSIONS).not.toContain('.txt');
});
it('never hides a file the upload would accept behind the picker filter', () => {
const listed = ATTACHMENT_FILE_ACCEPT_ATTR.split(',');
for (const ext of ATTACHMENT_PARSER_EXTENSIONS) {
expect(listed).toContain(ext);
}
// Parserless text (.txt, .py, .log) is accepted on content, so the
// picker must not filter it out.
expect(listed).toContain('.txt');
expect(listed).toContain('text/*');
});
});
describe('looksLikeText', () => {
it('accepts empty, plain, accented and ANSI-coloured text', () => {
expect(looksLikeText(new Uint8Array())).toBe(true);
expect(looksLikeText(new TextEncoder().encode('plain text\n'))).toBe(true);
expect(looksLikeText(new TextEncoder().encode('café — em dash\n'))).toBe(
true,
);
expect(looksLikeText(bytes(0x1b, 0x5b, 0x33, 0x31, 0x6d, 0x6f, 0x6b))).toBe(
true,
);
});
it('accepts BOM-marked Unicode, which is half NUL bytes but still text', () => {
// UTF-8, UTF-16 LE/BE, and UTF-32 BE (no TextDecoder — left to the server).
expect(looksLikeText(bytes(0xef, 0xbb, 0xbf, 0x68, 0x69))).toBe(true);
expect(looksLikeText(bytes(0xff, 0xfe, 0x68, 0x00, 0x69, 0x00))).toBe(true);
expect(looksLikeText(bytes(0xfe, 0xff, 0x00, 0x68, 0x00, 0x69))).toBe(true);
expect(looksLikeText(bytes(0x00, 0x00, 0xfe, 0xff, 0x00, 0x68))).toBe(true);
});
it('still rejects binary behind a BOM — three bytes buy nothing', () => {
// UTF-8 BOM + MP4 header: the byte rules apply to what follows it.
expect(looksLikeText(bytes(0xef, 0xbb, 0xbf, ...MP4_HEADER))).toBe(false);
// UTF-16 LE BOM + control-only code points.
expect(
looksLikeText(
bytes(0xff, 0xfe, 0x01, 0x00, 0x02, 0x00, 0x03, 0x00, 0x04, 0x00),
),
).toBe(false);
// UTF-16 BE BOM + bytes that decode to NUL characters.
expect(looksLikeText(bytes(0xfe, 0xff, 0x00, 0x00, 0x00, 0x41))).toBe(
false,
);
});
it('rejects a NUL byte and dense control bytes', () => {
expect(looksLikeText(MP4_HEADER)).toBe(false);
expect(looksLikeText(bytes(0x74, 0x78, 0x00, 0x74))).toBe(false);
expect(
looksLikeText(
new Uint8Array(Array.from({ length: 64 }, (_, i) => i + 1)),
),
).toBe(false);
});
});
describe('partitionAttachmentFiles', () => {
it('refuses video and archives whatever the picker claims the type is', async () => {
const { supported, unsupported } = await partitionAttachmentFiles([
file('clip.mp4', MP4_HEADER, 'text/plain'),
file('archive.zip', bytes(0x50, 0x4b, 0x03, 0x04, 0x00, 0x00)),
]);
expect(supported).toEqual([]);
expect(unsupported.map((f) => f.name)).toEqual(['clip.mp4', 'archive.zip']);
});
it('keeps text files that have no parser — the backend reads them', async () => {
const { supported, unsupported } = await partitionAttachmentFiles([
file('notes.txt', 'plain\n'),
file('main.py', 'def main():\n return 1\n'),
file('server.log', '2026-09-02 ERROR boom\n'),
file('Dockerfile', 'FROM python:3.12\n'),
]);
expect(supported.map((f) => f.name)).toEqual([
'notes.txt',
'main.py',
'server.log',
'Dockerfile',
]);
expect(unsupported).toEqual([]);
});
it('refuses a binary renamed to .txt — .txt is sniffed like any other suffix', async () => {
const { supported, unsupported } = await partitionAttachmentFiles([
file('notes.txt', MP4_HEADER),
// A BOM in front of it changes nothing.
file('bom.txt', bytes(0xef, 0xbb, 0xbf, ...MP4_HEADER)),
]);
expect(supported).toEqual([]);
expect(unsupported.map((f) => f.name)).toEqual(['notes.txt', 'bom.txt']);
});
it('admits parser-backed types on their suffix, binary contents and all', async () => {
// A PDF is binary; the sniff must never be applied to it.
const { supported, unsupported } = await partitionAttachmentFiles([
file('paper.pdf', MP4_HEADER),
file('photo.jpeg', MP4_HEADER),
]);
expect(supported.map((f) => f.name)).toEqual(['paper.pdf', 'photo.jpeg']);
expect(unsupported).toEqual([]);
});
it('partitions a mixed selection and keeps the original order', async () => {
const mp4 = file('clip.mp4', MP4_HEADER);
const txt = file('notes.txt');
const pdf = file('paper.pdf');
const { supported, unsupported } = await partitionAttachmentFiles([
mp4,
txt,
pdf,
]);
expect(supported).toEqual([txt, pdf]);
expect(unsupported).toEqual([mp4]);
});
});
describe('parseUploadErrorMessage', () => {
it('returns the server message when present', () => {
expect(
parseUploadErrorMessage(
'{"success":false,"message":"Unsupported file type: .mp4"}',
),
).toBe('Unsupported file type: .mp4');
});
it('returns undefined for non-JSON bodies or a missing message', () => {
expect(parseUploadErrorMessage('<html>502</html>')).toBeUndefined();
expect(parseUploadErrorMessage('{"ok":1}')).toBeUndefined();
expect(parseUploadErrorMessage('')).toBeUndefined();
});
});
describe('parseUploadErrorsByIndex', () => {
it('keys each reason by its upload_index', () => {
const byIndex = parseUploadErrorsByIndex(
JSON.stringify({
success: false,
message: 'Unsupported file type: .mp4',
errors: [
{
upload_index: 0,
filename: 'clip.mp4',
error: 'Unsupported file type: .mp4',
},
{
upload_index: 1,
filename: 'a.zip',
error: 'Unsupported file type: .zip',
},
],
}),
);
// Without this, both chips would repeat the first file's reason.
expect(byIndex.get(0)).toBe('Unsupported file type: .mp4');
expect(byIndex.get(1)).toBe('Unsupported file type: .zip');
});
it('is empty for bodies with no usable errors array', () => {
expect(parseUploadErrorsByIndex('<html>502</html>').size).toBe(0);
expect(parseUploadErrorsByIndex('{"message":"nope"}').size).toBe(0);
expect(parseUploadErrorsByIndex('').size).toBe(0);
expect(
parseUploadErrorsByIndex('{"errors":[{"error":"no index"}]}').size,
).toBe(0);
});
});
+243
View File
@@ -129,3 +129,246 @@ export const SOURCE_FILE_TREE_ACCEPT_ATTR = [
'.jpg',
'.jpeg',
].join(',');
/**
* Chat-attachment suffixes with a dedicated parser. Mirrors the backend's
* `ATTACHMENT_PARSER_EXTENSIONS` (application/parser/file/constants.py) —
* update both together. Zip is absent: source ingestion extracts archives,
* the attachment path does not.
*
* Not the whole allow-list, and `.txt` is deliberately not here: a suffix
* that isn't listed (.txt, .py, .log, .yaml) is read by the backend's
* plain-text fallthrough, so it is judged on content by
* `partitionAttachmentFiles` instead, exactly as the server does.
*/
export const ATTACHMENT_PARSER_EXTENSIONS: readonly string[] = [
'.rst',
'.md',
'.mdx',
'.pdf',
'.docx',
'.csv',
'.epub',
'.html',
'.xhtml',
'.json',
'.xlsx',
'.pptx',
// Read by the anydoc engine, a core backend dependency: legacy and
// macro-enabled Office, OpenDocument, RTF.
'.docm',
'.doc',
'.odt',
'.rtf',
'.pptm',
'.ppt',
'.pps',
'.ppsx',
'.ppsm',
'.pot',
'.odp',
'.xlsm',
'.xlsb',
'.xls',
'.ods',
'.adoc',
'.asciidoc',
'.png',
'.jpg',
'.jpeg',
'.tiff',
'.tif',
'.bmp',
'.webp',
'.vtt',
'.xml',
'.wav',
'.mp3',
'.m4a',
'.ogg',
'.webm',
];
/**
* Picker filter for the Attach button. A hint only — pickers may ignore it,
* and it must never be narrower than what the upload accepts: `text/*` (plus
* `.txt` explicitly) keeps parserless text files such as .py and .log
* selectable, since the gate reads those happily.
*/
export const ATTACHMENT_FILE_ACCEPT_ATTR = [
...ATTACHMENT_PARSER_EXTENSIONS,
'.txt',
'text/*',
].join(',');
/** Lower-cased last extension including the dot, or '' (dotfiles have none). */
export function getFileExtension(name: string): string {
const base = name.split(/[\\/]/).pop() ?? '';
const dot = base.lastIndexOf('.');
if (dot <= 0) return '';
return base.slice(dot).toLowerCase();
}
/**
* Whether the backend has a parser for this suffix. False is not a refusal:
* the file then has to read as text. Never decided by the mime type the
* picker reports — mobile pickers ignore `accept` and lie about types.
*/
export function hasAttachmentParser(file: { name: string }): boolean {
const ext = getFileExtension(file.name);
return ext !== '' && ATTACHMENT_PARSER_EXTENSIONS.includes(ext);
}
// Mirrors application/upload_limits.py: enough of the head to recognise a
// container header, and a tolerance that keeps real text (UTF-8 accents, an
// ANSI-coloured log) in while a random binary stays out.
const TEXT_SNIFF_BYTES = 8192;
const MAX_NONTEXT_RATIO = 0.1;
// Control bytes that occur in ordinary text: tab, LF, VT, FF, CR, ESC.
const TEXT_CONTROL_BYTES = new Set([0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x1b]);
const TEXT_CONTROL_CHARS = new Set(['\t', '\n', '\v', '\f', '\r', '\u001b']);
// \p{C}: control, format, private-use, surrogate and unassigned code points —
// what binary decodes into.
const NONTEXT_CHAR = /\p{C}/u;
// A UTF-16/32 file is half NUL bytes, so the byte rules below would reject it.
// Its BOM says which encoding to read it as; the decoded characters are then
// judged instead. Longest BOM first — UTF-32-LE starts with the UTF-16-LE one.
// A BOM is never a verdict on its own: three bytes must not buy a video a pass.
const TEXT_BOMS: { bom: number[]; encoding: string | null }[] = [
// TextDecoder has no UTF-32; leave those two to the server, which does.
{ bom: [0xff, 0xfe, 0x00, 0x00], encoding: null },
{ bom: [0x00, 0x00, 0xfe, 0xff], encoding: null },
{ bom: [0xef, 0xbb, 0xbf], encoding: 'utf-8' },
{ bom: [0xff, 0xfe], encoding: 'utf-16le' },
{ bom: [0xfe, 0xff], encoding: 'utf-16be' },
];
/**
* Whether decoded characters read as text rather than decoded binary.
* Replacement characters count against it, so bytes the decoder could not
* read are evidence rather than silently dropped.
*/
function decodedLooksLikeText(text: string): boolean {
// The byte-level NUL rule, one level up: no text holds a NUL character.
if (text.includes('\u0000')) return false;
let total = 0;
let nontext = 0;
for (const char of text) {
total += 1;
if (
char === '\ufffd' ||
(!TEXT_CONTROL_CHARS.has(char) && NONTEXT_CHAR.test(char))
)
nontext += 1;
}
if (total === 0) return true;
return nontext / total <= MAX_NONTEXT_RATIO;
}
/**
* Whether a leading byte sample reads as text rather than binary. A UTF-16
* BOM switches the test to the decoded characters; the content is still what
* decides. Mirrors `looks_like_text` in application/upload_limits.py.
*/
export function looksLikeText(sample: Uint8Array): boolean {
if (sample.length === 0) return true;
let body = sample;
const marked = TEXT_BOMS.find(({ bom }) =>
bom.every((byte, i) => sample[i] === byte),
);
if (marked) {
// UTF-32, or a decoder this browser lacks: defer to the server rather
// than refuse a file it would accept.
if (marked.encoding === null) return true;
body = sample.subarray(marked.bom.length);
if (body.length === 0) return true;
if (marked.encoding !== 'utf-8') {
try {
return decodedLooksLikeText(
new TextDecoder(marked.encoding).decode(body),
);
} catch {
return true;
}
}
// UTF-8 BOM: the byte rules still apply to everything after it.
}
let nontext = 0;
for (const byte of body) {
if (byte === 0x00) return false;
if ((byte < 0x20 && !TEXT_CONTROL_BYTES.has(byte)) || byte === 0x7f)
nontext += 1;
}
return nontext / body.length <= MAX_NONTEXT_RATIO;
}
async function isSupportedAttachmentFile(file: File): Promise<boolean> {
if (hasAttachmentParser(file)) return true;
try {
const head = await file.slice(0, TEXT_SNIFF_BYTES).arrayBuffer();
return looksLikeText(new Uint8Array(head));
} catch {
// Can't read it here — let the upload run and the server decide.
return true;
}
}
/**
* Split a selection into what the backend can read and what it will refuse,
* preserving order. Files with a parser pass on their suffix; the rest are
* sniffed, so source, config and log files go through and a video does not.
*/
export async function partitionAttachmentFiles(files: File[]): Promise<{
supported: File[];
unsupported: File[];
}> {
const verdicts = await Promise.all(files.map(isSupportedAttachmentFile));
const supported: File[] = [];
const unsupported: File[] = [];
files.forEach((file, index) => {
(verdicts[index] ? supported : unsupported).push(file);
});
return { supported, unsupported };
}
/** The ``message`` of a JSON error body, if the server sent one. */
export function parseUploadErrorMessage(body: string): string | undefined {
if (!body) return undefined;
try {
const parsed = JSON.parse(body) as { message?: unknown };
return typeof parsed?.message === 'string' && parsed.message
? parsed.message
: undefined;
} catch {
return undefined;
}
}
/**
* Per-file reasons from an error body, keyed by `upload_index`. A rejected
* batch carries one `errors` entry per file, so each chip can say why it
* failed instead of every chip repeating the first file's reason.
*/
export function parseUploadErrorsByIndex(body: string): Map<number, string> {
const byIndex = new Map<number, string>();
if (!body) return byIndex;
try {
const parsed = JSON.parse(body) as { errors?: unknown };
if (!Array.isArray(parsed?.errors)) return byIndex;
for (const entry of parsed.errors as {
upload_index?: unknown;
error?: unknown;
}[]) {
if (
typeof entry?.upload_index === 'number' &&
typeof entry.error === 'string'
)
byIndex.set(entry.upload_index, entry.error);
}
} catch {
// Not JSON (a proxy's HTML 502, say) — the caller falls back.
}
return byIndex;
}
+2 -1
View File
@@ -1101,7 +1101,8 @@
"waitingToSend_one": "Will send when {{count}} file finishes processing…",
"waitingToSend_other": "Will send when {{count}} files finish processing…",
"cancelQueuedSend": "Cancel",
"sendBlockedByFailed": "{{names}} couldn't be processed — remove to send"
"sendBlockedByFailed": "{{names}} couldn't be processed — remove to send",
"unsupportedType": "Not a supported file type ({{extension}})"
},
"retry": "Retry",
"reasoning": "Reasoning",
+2
View File
@@ -46,6 +46,8 @@ export interface Attachment {
*/
attachmentId?: string;
token_count?: number;
/** Why a ``failed`` attachment failed, when known (server or client gate). */
errorMessage?: string;
}
export type UploadTaskStatus =