Merge branch 'main' into anydoc-support

Conflicts, and how each was taken:

- application/core/settings.py — ours. The renamed OCR_ENABLED /
  OCR_ATTACHMENTS_ENABLED / OCR_MIN_CHARS_PER_PAGE accept main's
  DOCLING_OCR_* spellings as AliasChoices, so nothing is dropped.
- application/Dockerfile — both. Main's install layers plus the
  INSTALL_DOCLING build arg.
- application/parser/file/constants.py — both imports.
- deployment/docker-compose.yaml — both. The INSTALL_DOCLING /
  INSTALL_TESSERACT build args on backend and worker, and main's
  -Q docsgpt,parsing,embeddings, which query embedding needs.
- tests/conftest.py — theirs. Both sides fixed the same pytest-postgresql
  9.0.0 autocommit= breakage; main's spelling is the one already on main.
- application/requirements.txt — the comments claimed different reasons
  torch is in core. Main's is the true one now: it removed
  sentence-transformers, so docling is torch's only remaining consumer.

Two things the merge broke without conflicting:

- onnxruntime. This branch moved it out of core into the docling extra;
  main meanwhile made it the runtime local embeddings execute on
  (fastembed). Git took the deletion, leaving fastembed with no pinned
  runtime in a repo that pins everything. Restored to core, and no longer
  pinned twice from the extra.
- The frontend copy of ATTACHMENT_PARSER_EXTENSIONS. The backend list is
  derived and picked up the anydoc suffixes; the hand-kept frontend mirror
  did not, so the composer would refuse files the API accepts.
  tests/parser/file/test_constants.py is what caught it.

ruff, pytest (9897 passed), frontend build and docs build all pass. The
image build is unverified: no Docker daemon on this machine.
This commit is contained in:
Alex committed 2026-09-04 16:42:48 +01:00
commit 3947c66cda
97 files changed
+8661 -1049

No files matched your search

+109
View File
@@ -0,0 +1,109 @@
"""Tests for the shared file-extension constants."""
import re
from importlib.util import find_spec
from pathlib import Path
import pytest
from application.parser.file.bulk import get_default_file_extractor
from application.parser.file.constants import (
ATTACHMENT_PARSER_EXTENSIONS,
attachment_extension,
has_attachment_parser,
SUPPORTED_SOURCE_EXTENSIONS,
)
FRONTEND_CONSTANTS = (
Path(__file__).resolve().parents[3] / "frontend/src/constants/fileUpload.ts"
)
@pytest.mark.unit
def test_parser_extensions_match_the_extractor():
"""The list is a hand-kept literal; this is what keeps it honest.
``ATTACHMENT_PARSER_EXTENSIONS`` cannot import the extractor (that would
pull docling into the API process), so a parser added to
``get_default_file_extractor`` would otherwise stay refused as an
attachment — which is how .webp, .tiff, .vtt and .xml were first missed.
A suffix listed here but *without* a parser is the opposite failure: it
skips the content check and reaches the plain-text fallthrough, which is
how a binary named notes.txt got through.
"""
parser_suffixes = set(get_default_file_extractor())
assert parser_suffixes <= set(ATTACHMENT_PARSER_EXTENSIONS), (
"parsers exist for suffixes the attachment gate refuses: "
f"{sorted(parser_suffixes - set(ATTACHMENT_PARSER_EXTENSIONS))}"
)
if find_spec("docling") is not None:
assert parser_suffixes == set(ATTACHMENT_PARSER_EXTENSIONS), (
"listed as parser-backed but nothing parses them: "
f"{sorted(set(ATTACHMENT_PARSER_EXTENSIONS) - parser_suffixes)}"
)
@pytest.mark.unit
def test_frontend_mirrors_the_parser_extensions():
"""The composer gates uploads client-side; a divergent list refuses valid files.
``ATTACHMENT_PARSER_EXTENSIONS`` in the frontend is a hand-kept copy of
the backend's. Nothing but this test connects them, and .mdx, .xhtml and
.adoc were blocked in the UI while the API accepted them before it
existed.
"""
if not FRONTEND_CONSTANTS.exists():
pytest.skip("frontend sources not present in this checkout")
source = FRONTEND_CONSTANTS.read_text(encoding="utf-8")
match = re.search(
r"ATTACHMENT_PARSER_EXTENSIONS:\s*readonly string\[\]\s*=\s*\[(.*?)\]",
source,
re.DOTALL,
)
assert match, "ATTACHMENT_PARSER_EXTENSIONS not found in fileUpload.ts"
frontend_extensions = set(re.findall(r"'(\.[^']+)'", match.group(1)))
assert frontend_extensions == set(ATTACHMENT_PARSER_EXTENSIONS), (
"frontend and backend attachment lists disagree — "
f"backend only: {sorted(set(ATTACHMENT_PARSER_EXTENSIONS) - frontend_extensions)}, "
f"frontend only: {sorted(frontend_extensions - set(ATTACHMENT_PARSER_EXTENSIONS))}"
)
@pytest.mark.unit
def test_source_ingestion_types_are_parser_backed_apart_from_text():
"""Source ingestion's own list must not smuggle a parserless suffix past the sniff."""
assert set(SUPPORTED_SOURCE_EXTENSIONS) - set(ATTACHMENT_PARSER_EXTENSIONS) == {
".txt"
}
@pytest.mark.unit
@pytest.mark.parametrize(
("filename", "expected"),
[
("Photo.JPG", ".jpg"),
("/tmp/dir.d/report.PDF", ".pdf"),
("archive.tar.gz", ".gz"),
("Dockerfile", ""),
(".env", ""),
("trailing.", "."),
(None, ""),
],
)
def test_attachment_extension(filename, expected):
assert attachment_extension(filename) == expected
@pytest.mark.unit
def test_has_attachment_parser_is_case_insensitive_and_excludes_zip():
assert has_attachment_parser("Report.PDF")
assert has_attachment_parser("scan.WebP")
# No parser: admitted (or not) on content instead, never on the suffix.
assert not has_attachment_parser("archive.zip")
assert not has_attachment_parser("main.py")
assert not has_attachment_parser("clip.mp4")
# .txt is the plain-text fallthrough itself, not a parser.
assert not has_attachment_parser("notes.txt")