Merge branch 'main' into anydoc-support

Conflicts, and how each was taken:

- application/core/settings.py — ours. The renamed OCR_ENABLED /
  OCR_ATTACHMENTS_ENABLED / OCR_MIN_CHARS_PER_PAGE accept main's
  DOCLING_OCR_* spellings as AliasChoices, so nothing is dropped.
- application/Dockerfile — both. Main's install layers plus the
  INSTALL_DOCLING build arg.
- application/parser/file/constants.py — both imports.
- deployment/docker-compose.yaml — both. The INSTALL_DOCLING /
  INSTALL_TESSERACT build args on backend and worker, and main's
  -Q docsgpt,parsing,embeddings, which query embedding needs.
- tests/conftest.py — theirs. Both sides fixed the same pytest-postgresql
  9.0.0 autocommit= breakage; main's spelling is the one already on main.
- application/requirements.txt — the comments claimed different reasons
  torch is in core. Main's is the true one now: it removed
  sentence-transformers, so docling is torch's only remaining consumer.

Two things the merge broke without conflicting:

- onnxruntime. This branch moved it out of core into the docling extra;
  main meanwhile made it the runtime local embeddings execute on
  (fastembed). Git took the deletion, leaving fastembed with no pinned
  runtime in a repo that pins everything. Restored to core, and no longer
  pinned twice from the extra.
- The frontend copy of ATTACHMENT_PARSER_EXTENSIONS. The backend list is
  derived and picked up the anydoc suffixes; the hand-kept frontend mirror
  did not, so the composer would refuse files the API accepts.
  tests/parser/file/test_constants.py is what caught it.

ruff, pytest (9897 passed), frontend build and docs build all pass. The
image build is unverified: no Docker daemon on this machine.

tests/parser/conftest.py

@@ -0,0 +1,23 @@
+import pytest
+
+
+@pytest.fixture(scope="session")
+def _mpnet_tokenizer():
+    """Real WordPiece tokenizer; skipped when the hub is unreachable."""
+    pytest.importorskip("tokenizers")
+    from tokenizers import Tokenizer
+
+    try:
+        tokenizer = Tokenizer.from_pretrained("sentence-transformers/all-mpnet-base-v2")
+    except Exception as exc:  # offline CI
+        pytest.skip(f"tokenizer unavailable: {exc}")
+    tokenizer.no_padding()
+    tokenizer.no_truncation()
+    return tokenizer
+
+
+@pytest.fixture
+def hf_counter(_mpnet_tokenizer):
+    from application.parser.tokenization import HuggingFaceCounter
+
+    return HuggingFaceCounter(_mpnet_tokenizer, "sentence-transformers/all-mpnet-base-v2")

tests/parser/file/test_constants.py

@@ -0,0 +1,109 @@
+"""Tests for the shared file-extension constants."""
+
+import re
+from importlib.util import find_spec
+from pathlib import Path
+
+import pytest
+
+from application.parser.file.bulk import get_default_file_extractor
+from application.parser.file.constants import (
+    ATTACHMENT_PARSER_EXTENSIONS,
+    attachment_extension,
+    has_attachment_parser,
+    SUPPORTED_SOURCE_EXTENSIONS,
+)
+
+FRONTEND_CONSTANTS = (
+    Path(__file__).resolve().parents[3] / "frontend/src/constants/fileUpload.ts"
+)
+
+
+@pytest.mark.unit
+def test_parser_extensions_match_the_extractor():
+    """The list is a hand-kept literal; this is what keeps it honest.
+
+    ``ATTACHMENT_PARSER_EXTENSIONS`` cannot import the extractor (that would
+    pull docling into the API process), so a parser added to
+    ``get_default_file_extractor`` would otherwise stay refused as an
+    attachment — which is how .webp, .tiff, .vtt and .xml were first missed.
+    A suffix listed here but *without* a parser is the opposite failure: it
+    skips the content check and reaches the plain-text fallthrough, which is
+    how a binary named notes.txt got through.
+    """
+    parser_suffixes = set(get_default_file_extractor())
+
+    assert parser_suffixes <= set(ATTACHMENT_PARSER_EXTENSIONS), (
+        "parsers exist for suffixes the attachment gate refuses: "
+        f"{sorted(parser_suffixes - set(ATTACHMENT_PARSER_EXTENSIONS))}"
+    )
+    if find_spec("docling") is not None:
+        assert parser_suffixes == set(ATTACHMENT_PARSER_EXTENSIONS), (
+            "listed as parser-backed but nothing parses them: "
+            f"{sorted(set(ATTACHMENT_PARSER_EXTENSIONS) - parser_suffixes)}"
+        )
+
+
+@pytest.mark.unit
+def test_frontend_mirrors_the_parser_extensions():
+    """The composer gates uploads client-side; a divergent list refuses valid files.
+
+    ``ATTACHMENT_PARSER_EXTENSIONS`` in the frontend is a hand-kept copy of
+    the backend's. Nothing but this test connects them, and .mdx, .xhtml and
+    .adoc were blocked in the UI while the API accepted them before it
+    existed.
+    """
+    if not FRONTEND_CONSTANTS.exists():
+        pytest.skip("frontend sources not present in this checkout")
+
+    source = FRONTEND_CONSTANTS.read_text(encoding="utf-8")
+    match = re.search(
+        r"ATTACHMENT_PARSER_EXTENSIONS:\s*readonly string\[\]\s*=\s*\[(.*?)\]",
+        source,
+        re.DOTALL,
+    )
+    assert match, "ATTACHMENT_PARSER_EXTENSIONS not found in fileUpload.ts"
+    frontend_extensions = set(re.findall(r"'(\.[^']+)'", match.group(1)))
+
+    assert frontend_extensions == set(ATTACHMENT_PARSER_EXTENSIONS), (
+        "frontend and backend attachment lists disagree — "
+        f"backend only: {sorted(set(ATTACHMENT_PARSER_EXTENSIONS) - frontend_extensions)}, "
+        f"frontend only: {sorted(frontend_extensions - set(ATTACHMENT_PARSER_EXTENSIONS))}"
+    )
+
+
+@pytest.mark.unit
+def test_source_ingestion_types_are_parser_backed_apart_from_text():
+    """Source ingestion's own list must not smuggle a parserless suffix past the sniff."""
+    assert set(SUPPORTED_SOURCE_EXTENSIONS) - set(ATTACHMENT_PARSER_EXTENSIONS) == {
+        ".txt"
+    }
+
+
+@pytest.mark.unit
+@pytest.mark.parametrize(
+    ("filename", "expected"),
+    [
+        ("Photo.JPG", ".jpg"),
+        ("/tmp/dir.d/report.PDF", ".pdf"),
+        ("archive.tar.gz", ".gz"),
+        ("Dockerfile", ""),
+        (".env", ""),
+        ("trailing.", "."),
+        (None, ""),
+    ],
+)
+def test_attachment_extension(filename, expected):
+    assert attachment_extension(filename) == expected
+
+
+@pytest.mark.unit
+def test_has_attachment_parser_is_case_insensitive_and_excludes_zip():
+    assert has_attachment_parser("Report.PDF")
+    assert has_attachment_parser("scan.WebP")
+    # No parser: admitted (or not) on content instead, never on the suffix.
+    assert not has_attachment_parser("archive.zip")
+    assert not has_attachment_parser("main.py")
+    assert not has_attachment_parser("clip.mp4")
+    # .txt is the plain-text fallthrough itself, not a parser.
+    assert not has_attachment_parser("notes.txt")

tests/parser/test_chunking.py

@@ -108,18 +108,89 @@ class TestSplitDocument:
 
         result = chunker.split_document(doc)
         assert len(result) > 1
-        # First chunk should contain header
         assert "h1" in result[0].text
+        # Only the first: this is what duplicate_headers=False means.
+        assert all("h1" not in chunk.text for chunk in result[1:])
 
     def test_split_duplicates_header(self):
+        """Every chunk carries the header when the flag is set.
+
+        Asserting only the first chunk passed even while the flag did nothing:
+        the implementation cleared the header after the first iteration, so
+        duplicate_headers was unreachable.
+        """
         chunker = Chunker(max_tokens=50, min_tokens=5, duplicate_headers=True)
         text = "h1\nh2\nh3\n" + "word " * 200
         doc = Document(text=text, doc_id="doc1")
 
         result = chunker.split_document(doc)
         assert len(result) > 1
-        # First chunk should contain header
-        assert "h1" in result[0].text
+        assert all("h1" in chunk.text for chunk in result)
+
+    def test_oversized_header_does_not_multiply_chunks(self):
+        """A header past the budget used to leave a one-token body budget.
+
+        ``max(1, max_tokens - header_tokens)`` bottomed out at 1, so the body
+        was cut into one chunk per token -- each still over ``max_tokens``,
+        since the whole header was prepended to it.
+        """
+        chunker = Chunker(max_tokens=50, min_tokens=5, duplicate_headers=True)
+        header = ("verylongheaderword " * 40) + "h2\nh3\n"
+        body = "word " * 100
+        doc = Document(text=f"{header}\n{body}", doc_id="doc1")
+
+        result = chunker.split_document(doc)
+
+        body_tokens = chunker.counter.count(body)
+        assert len(result) <= body_tokens // 10, "chunk count must track the budget"
+        for chunk in result:
+            assert chunk.extra_info["token_count"] <= chunker.max_tokens
+        assert "".join(chunk.text for chunk in result) == doc.text
+
+    def test_header_taking_most_of_the_budget_is_not_duplicated(self):
+        """Duplication is dropped when it would leave almost no room for body.
+
+        Below a quarter of the budget every chunk is mostly repeated header,
+        which multiplies the chunk count without adding retrievable text.
+        """
+        chunker = Chunker(max_tokens=50, min_tokens=5, duplicate_headers=True)
+        header = ("headerword " * 20) + "h2\nh3\n"
+        doc = Document(text=f"{header}\n" + "word " * 200, doc_id="doc1")
+
+        result = chunker.split_document(doc)
+
+        assert len(result) > 1
+        assert "headerword" in result[0].text
+        assert all("headerword" not in chunk.text for chunk in result[1:])
+
+    def test_header_only_document_is_not_dropped(self):
+        """The loop only ever emitted the header attached to a body piece.
+
+        With no body there was no piece, so an entire document disappeared
+        from the index with no error and no log line.
+        """
+        chunker = Chunker(max_tokens=50, min_tokens=1, duplicate_headers=False)
+        text = "h1\nh2\nh3\n"
+        doc = Document(text=text, doc_id="doc1")
+
+        result = chunker.split_document(doc)
+
+        assert len(result) == 1
+        assert result[0].text == text
+
+    def test_header_only_document_keeps_its_text_through_chunk(self):
+        """The reachable shape: three lines that alone exceed the budget."""
+        chunker = Chunker(max_tokens=5, min_tokens=1, duplicate_headers=False)
+        text = "h1\nh2\nh3\n"
+
+        result = chunker.chunk([Document(text=text, doc_id="doc1")])
+
+        assert result, "the document must not vanish"
+        assert "".join(chunk.text for chunk in result) == text
+
+    def test_zero_max_tokens_does_not_split_per_token(self):
+        chunker = Chunker(max_tokens=0, min_tokens=1)
+        assert chunker.max_tokens == 1
 
     def test_split_preserves_embedding(self):
         chunker = Chunker(max_tokens=50, min_tokens=5)

tests/parser/test_chunking_strategies.py

@@ -15,11 +15,13 @@ from application.parser.chunking_strategies import (
     SemanticChunker,
 )
 from application.parser.schema.base import Document
-from application.utils import get_encoding
+from application.parser.tokenization import get_token_counter
 
 
 def _tok(text: str) -> int:
-    return len(get_encoding().encode(text))
+    # Chunk budgets are enforced in the embedding model's tokenizer, so a
+    # test that measures them in cl100k measures the wrong thing.
+    return get_token_counter().count(text)
 
 
 @pytest.mark.unit

tests/parser/test_tokenization.py

@@ -0,0 +1,300 @@
+"""Chunk sizes must be counted in the embedding model's units, and splitting
+must never rewrite the text it splits."""
+
+import pytest
+
+from application.parser import tokenization
+from application.parser.tokenization import (
+    HuggingFaceCounter,
+    TiktokenCounter,
+    get_token_counter,
+)
+
+SAMPLES = [
+    "Hello World: DocsGPT ANSWERS Questions.",
+    "The quick brown fox jumps over the lazy dog. " * 40,
+    "Comment configurer l'authentification avec une clé API ?",
+    "def embed(text: str) -> list[float]:\n    return model.encode(text)\n",
+    "Ünïcödé — em-dashes, curly “quotes”, and 日本語 text.",
+    "a,b,c\n1,2,3\n4,5,6\n" * 30,
+]
+
+
+class _StubEncoding:
+    """Whitespace tokenizer standing in for tiktoken."""
+
+    def encode_ordinary(self, text):
+        return [ord(c) for c in text]
+
+    def decode(self, ids):
+        return "".join(chr(i) for i in ids)
+
+    def decode_with_offsets(self, ids):
+        # One token per character, so each token starts where the last ended.
+        return self.decode(ids), list(range(len(ids)))
+
+
+@pytest.fixture(autouse=True)
+def _clear_cache():
+    tokenization.reset_cache()
+    yield
+    tokenization.reset_cache()
+
+
+class TestSplittingPreservesText:
+    """The property that protects every stored document."""
+
+    @pytest.mark.parametrize("text", SAMPLES)
+    def test_tiktoken_split_reassembles_exactly(self, text, monkeypatch):
+        monkeypatch.setattr(tokenization, "get_encoding", _StubEncoding)
+        counter = TiktokenCounter()
+        pieces = counter.split(text, 7)
+        assert "".join(pieces) == text
+
+    @pytest.mark.parametrize("text", SAMPLES)
+    def test_hf_split_reassembles_exactly(self, text, hf_counter):
+        """WordPiece lowercases on decode, so splitting must slice, not decode."""
+        pieces = hf_counter.split(text, 7)
+        assert "".join(pieces) == text
+
+    def test_hf_split_does_not_lowercase(self, hf_counter):
+        text = "Hello World: DocsGPT ANSWERS Questions."
+        assert "".join(hf_counter.split(text, 3)) == text
+        assert "DocsGPT" in "".join(hf_counter.split(text, 3))
+
+    @pytest.mark.parametrize("text", SAMPLES)
+    def test_every_piece_is_within_budget(self, text, hf_counter):
+        budget = 10
+        for piece in hf_counter.split(text, budget):
+            # The final piece can absorb trailing characters the tokenizer
+            # dropped, so allow a small overshoot there only.
+            assert hf_counter.count(piece) <= budget + 2
+
+    def test_short_text_is_returned_whole(self, hf_counter):
+        assert hf_counter.split("short", 100) == ["short"]
+
+    def test_empty_text_yields_no_pieces(self, hf_counter):
+        assert hf_counter.split("", 10) == []
+
+    def test_zero_budget_is_clamped_not_infinite_loop(self, hf_counter):
+        pieces = hf_counter.split("some words here to split", 0)
+        assert "".join(pieces) == "some words here to split"
+
+
+class TestCounting:
+    def test_counts_differ_between_tokenizers(self, hf_counter, monkeypatch):
+        """The whole point: mpnet and cl100k disagree, so units matter."""
+        monkeypatch.setattr(tokenization, "get_encoding", _StubEncoding)
+        text = "internationalisation tokenization"
+        assert hf_counter.count(text) != TiktokenCounter().count(text)
+
+    def test_empty_text_counts_zero(self, hf_counter):
+        assert hf_counter.count("") == 0
+
+
+class TestSelection:
+    def test_registered_model_uses_its_own_tokenizer(self):
+        counter = get_token_counter("huggingface_sentence-transformers/all-mpnet-base-v2")
+        assert isinstance(counter, HuggingFaceCounter)
+        assert counter.name == "sentence-transformers/all-mpnet-base-v2"
+
+    def test_openai_model_falls_back_to_cl100k(self):
+        """OpenAI models are served remotely and genuinely count cl100k."""
+        assert isinstance(get_token_counter("openai_text-embedding-ada-002"), TiktokenCounter)
+
+    def test_unreachable_tokenizer_falls_back_rather_than_raising(self, monkeypatch):
+        monkeypatch.setattr(tokenization, "_load_hf_counter", lambda repo: None)
+        assert isinstance(get_token_counter("granite-311m"), TiktokenCounter)
+
+    def test_counter_is_cached_per_model(self):
+        first = get_token_counter("granite-311m")
+        assert get_token_counter("granite-311m") is first
+
+
+class TestTiktokenSplitAgainstRealCl100k:
+    """The stub above is one token per character, so it can never place a cut
+    inside a character. Real cl100k can, and that is the case that corrupted
+    text: decoding each window on its own turns a straddled multi-byte
+    character into U+FFFD on both sides of the cut."""
+
+    @pytest.fixture
+    def real_counter(self):
+        try:
+            counter = TiktokenCounter()
+            counter.count("probe")
+        except Exception as exc:  # offline CI, same policy as the HF fixture
+            pytest.skip(f"cl100k encoding unavailable: {exc}")
+        return counter
+
+    # 2000 is the shipped default max_tokens, 384 mpnet's window; the small
+    # values place many more cuts per unit of text.
+    @pytest.mark.parametrize("window", [1, 2, 3, 7, 128, 384, 2000])
+    @pytest.mark.parametrize(
+        "text",
+        [
+            "日本語のテキストです。絵文字も🎉あります。",
+            "検索は自然言語でできます。" * 200,
+            "Здравствуйте, как настроить аутентификацию?",
+            "🎉🎊✨🚀🔥💡📚🧠" * 50,
+            "Ünïcödé — em-dashes, curly “quotes”, and 日本語 text.",
+        ],
+        ids=["ja-short", "ja-long", "ru", "emoji", "mixed"],
+    )
+    def test_split_reassembles_exactly(self, real_counter, text, window):
+        pieces = real_counter.split(text, window)
+        assert "".join(pieces) == text
+
+    @pytest.mark.parametrize("window", [1, 3, 128, 2000])
+    def test_split_never_emits_a_replacement_character(self, real_counter, window):
+        text = "検索は自然言語でできます。絵文字も🎉あります。" * 100
+        assert "�" not in "".join(real_counter.split(text, window))
+
+    def test_first_window_budget_is_honoured_and_lossless(self, real_counter):
+        text = "日本語のテキストです。" * 50
+        pieces = real_counter.split(text, 20, first_max_tokens=5)
+        assert "".join(pieces) == text
+        assert real_counter.count(pieces[0]) <= 5
+
+
+class TestTiktokenCounterEdges:
+    """The cl100k path is the fallback, so its edges matter as much."""
+
+    @pytest.fixture
+    def counter(self, monkeypatch):
+        monkeypatch.setattr(tokenization, "get_encoding", _StubEncoding)
+        return TiktokenCounter()
+
+    def test_empty_text_counts_zero_and_splits_to_nothing(self, counter):
+        assert counter.count("") == 0
+        assert counter.split("", 10) == []
+
+    def test_text_within_budget_is_returned_whole(self, counter):
+        assert counter.split("abc", 10) == ["abc"]
+
+    def test_first_window_can_be_smaller_than_the_rest(self, counter):
+        """A header eats into the first chunk's budget only."""
+        pieces = counter.split("abcdefghij", 4, first_max_tokens=2)
+        assert pieces[0] == "ab"
+        assert "".join(pieces) == "abcdefghij"
+
+
+class TestCounterContract:
+    def test_base_class_requires_an_implementation(self):
+        base = tokenization.TokenCounter()
+        with pytest.raises(NotImplementedError):
+            base.count("x")
+        with pytest.raises(NotImplementedError):
+            base.split("x", 1)
+
+
+class TestFallbackWhenTokenizerUnavailable:
+    def test_load_failure_returns_none_rather_than_raising(self, monkeypatch, caplog):
+        """Chunking must survive an offline host or a bad repo name."""
+        import builtins
+
+        real_import = builtins.__import__
+
+        def boom(name, *args, **kwargs):
+            if name == "tokenizers":
+                raise ImportError("no tokenizers here")
+            return real_import(name, *args, **kwargs)
+
+        monkeypatch.setattr(builtins, "__import__", boom)
+        assert tokenization._load_hf_counter("some/repo") is None
+
+    def test_selection_falls_back_to_cl100k_on_failure(self, monkeypatch):
+        monkeypatch.setattr(tokenization, "_load_hf_counter", lambda repo: None)
+        assert isinstance(get_token_counter("granite-97m"), TiktokenCounter)
+
+    def test_reset_cache_forces_reselection(self, monkeypatch):
+        first = get_token_counter("granite-311m")
+        tokenization.reset_cache()
+        monkeypatch.setattr(tokenization, "_load_hf_counter", lambda repo: None)
+        assert get_token_counter("granite-311m") is not first
+
+
+class TestOffsetsWithoutSpans:
+    """Some tokenizers emit ``(0, 0)`` for specials or normalised-away chars.
+
+    Those tokens consume budget but point at no text, so the splitter has to
+    skip the window rather than emit an empty piece or lose the tail.
+    """
+
+    class _Encoded:
+        def __init__(self, offsets):
+            self.offsets = offsets
+            self.ids = list(range(len(offsets)))
+
+    class _Tokenizer:
+        def __init__(self, offsets):
+            self._offsets = offsets
+
+        def encode(self, text, add_special_tokens=False):
+            return TestOffsetsWithoutSpans._Encoded(self._offsets)
+
+    def _counter(self, offsets):
+        return HuggingFaceCounter(self._Tokenizer(offsets), "stub")
+
+    def test_span_less_windows_are_skipped_not_emitted_empty(self):
+        # Two real tokens, then a window of pure (0, 0) padding-like entries.
+        counter = self._counter([(0, 2), (2, 4), (0, 0), (0, 0)])
+        pieces = counter.split("abcd", 2)
+        assert "" not in pieces
+        assert "".join(pieces) == "abcd"
+
+    def test_trailing_text_is_never_dropped(self):
+        """Offsets that stop short of the string must not lose the remainder."""
+        counter = self._counter([(0, 1), (1, 2), (2, 3)])
+        pieces = counter.split("abcdef", 2)
+        assert "".join(pieces) == "abcdef"
+
+    def test_all_span_less_offsets_still_return_the_text(self):
+        counter = self._counter([(0, 0), (0, 0), (0, 0)])
+        assert "".join(counter.split("abc", 1)) == "abc"
+
+
+class TestUnknownTokenCollapse:
+    """A tokenizer that folds a long unbroken run into one ``[UNK]``.
+
+    WordPiece gives up on any word longer than ``max_input_chars_per_word``
+    and emits a single unknown token for it. Counting that as one token makes
+    a base64 blob or a minified bundle look tiny, so the chunker never splits
+    it and an oversized chunk reaches the embedding server.
+    """
+
+    class _CollapsingEncoding:
+        """One token per whitespace-separated word, however long the word."""
+
+        def __init__(self, text):
+            self.ids = []
+            self.offsets = []
+            cursor = 0
+            for word in text.split(" "):
+                if word:
+                    self.ids.append(0)
+                    self.offsets.append((cursor, cursor + len(word)))
+                cursor += len(word) + 1
+
+    class _CollapsingTokenizer:
+        def encode(self, text, add_special_tokens=False):
+            return TestUnknownTokenCollapse._CollapsingEncoding(text)
+
+    def _counter(self):
+        return tokenization.HuggingFaceCounter(self._CollapsingTokenizer(), "stub")
+
+    def test_long_unbroken_run_is_charged_by_its_span(self):
+        counter = self._counter()
+        assert counter.count("a" * 9000) > 100
+
+    def test_ordinary_prose_is_unaffected(self):
+        counter = self._counter()
+        text = "the quick brown fox jumps over the lazy dog"
+        assert counter.count(text) == 9
+
+    def test_split_bounds_a_collapsed_run(self):
+        counter = self._counter()
+        text = "a" * 9000
+        pieces = counter.split(text, 20)
+        assert "".join(pieces) == text, "split must not lose or alter text"
+        assert len(pieces) > 1, "a collapsed run must still be cut into pieces"
+        assert all(counter.count(p) <= 20 for p in pieces)