Merge branch 'main' into anydoc-support

Conflicts, and how each was taken:

- application/core/settings.py — ours. The renamed OCR_ENABLED /
  OCR_ATTACHMENTS_ENABLED / OCR_MIN_CHARS_PER_PAGE accept main's
  DOCLING_OCR_* spellings as AliasChoices, so nothing is dropped.
- application/Dockerfile — both. Main's install layers plus the
  INSTALL_DOCLING build arg.
- application/parser/file/constants.py — both imports.
- deployment/docker-compose.yaml — both. The INSTALL_DOCLING /
  INSTALL_TESSERACT build args on backend and worker, and main's
  -Q docsgpt,parsing,embeddings, which query embedding needs.
- tests/conftest.py — theirs. Both sides fixed the same pytest-postgresql
  9.0.0 autocommit= breakage; main's spelling is the one already on main.
- application/requirements.txt — the comments claimed different reasons
  torch is in core. Main's is the true one now: it removed
  sentence-transformers, so docling is torch's only remaining consumer.

Two things the merge broke without conflicting:

- onnxruntime. This branch moved it out of core into the docling extra;
  main meanwhile made it the runtime local embeddings execute on
  (fastembed). Git took the deletion, leaving fastembed with no pinned
  runtime in a repo that pins everything. Restored to core, and no longer
  pinned twice from the extra.
- The frontend copy of ATTACHMENT_PARSER_EXTENSIONS. The backend list is
  derived and picked up the anydoc suffixes; the hand-kept frontend mirror
  did not, so the composer would refuse files the API accepts.
  tests/parser/file/test_constants.py is what caught it.

ruff, pytest (9897 passed), frontend build and docs build all pass. The
image build is unverified: no Docker daemon on this machine.

tests/api/user/attachments/test_routes.py

@@ -2369,6 +2369,20 @@ class TestGetStoreAttachmentUserError:
         msg = _get_store_attachment_user_error(RuntimeError("oops"))
         assert msg == "Failed to process file"
 
+    @pytest.mark.unit
+    def test_unsupported_type_message_is_rebuilt_from_the_filename(self):
+        """Nothing is read off the exception, so its text cannot reach a response."""
+        from application.api.user.attachments.routes import (
+            _get_store_attachment_user_error,
+        )
+        from application.upload_limits import UnsupportedUploadTypeError
+
+        err = UnsupportedUploadTypeError("internals: /srv/app/tmp/staged-42")
+        msg = _get_store_attachment_user_error(err, "clip.mp4")
+
+        assert msg == "Unsupported file type: .mp4"
+        assert "/srv/app" not in msg
+
 
 class TestRequireLiveSttRedisUnavailable:
     """Cover lines 99-102: Redis unavailable returns 503."""
@@ -2386,3 +2400,120 @@ class TestRequireLiveSttRedisUnavailable:
                 result = _require_live_stt_redis()
                 assert hasattr(result, "status_code")
                 assert result.status_code == 503
+
+
+MP4_BYTES = b"\x00\x00\x00\x18ftypisom\x00\x00\x02\x00isomiso2avc1mp41\x00\x00\x00\x08free"
+
+
+class TestStoreAttachmentTypeGate:
+    """Unparseable uploads are refused before anything is stored or queued."""
+
+    @patch("application.api.user.tasks.store_attachment.delay")
+    def test_rejects_unsupported_file_type_before_storage(
+        self, mock_store_attachment, flask_app
+    ):
+        from application.api.user.attachments.routes import StoreAttachment
+
+        app = Flask(__name__)
+        mock_storage = MagicMock()
+        with patch("application.api.user.base.storage", mock_storage), app.test_request_context(
+            "/api/store_attachment",
+            method="POST",
+            data={"file": (io.BytesIO(MP4_BYTES), "clip.mp4")},
+            content_type="multipart/form-data",
+        ):
+            request.decoded_token = {"sub": "test_user"}
+            response = StoreAttachment().post()
+
+        assert _get_response_status(response) == 400
+        body = _get_response_json(response)
+        assert body["success"] is False
+        assert body["message"] == "Unsupported file type: .mp4"
+        assert body["errors"] == [
+            {"upload_index": 0, "filename": "clip.mp4", "error": "Unsupported file type: .mp4"}
+        ]
+        mock_storage.save_file.assert_not_called()
+        mock_store_attachment.assert_not_called()
+
+    @patch("application.api.user.tasks.store_attachment.delay")
+    def test_rejects_a_binary_renamed_to_a_text_suffix(
+        self, mock_store_attachment, flask_app
+    ):
+        """.txt has no parser, so it is judged on content like any other suffix."""
+        from application.api.user.attachments.routes import StoreAttachment
+
+        app = Flask(__name__)
+        mock_storage = MagicMock()
+        with patch("application.api.user.base.storage", mock_storage), app.test_request_context(
+            "/api/store_attachment",
+            method="POST",
+            data={"file": (io.BytesIO(MP4_BYTES), "notes.txt")},
+            content_type="multipart/form-data",
+        ):
+            request.decoded_token = {"sub": "test_user"}
+            response = StoreAttachment().post()
+
+        assert _get_response_status(response) == 400
+        assert _get_response_json(response)["message"] == "Unsupported file type: .txt"
+        mock_storage.save_file.assert_not_called()
+        mock_store_attachment.assert_not_called()
+
+    @patch("application.api.user.tasks.store_attachment.delay")
+    def test_accepts_a_text_file_with_no_dedicated_parser(
+        self, mock_store_attachment, flask_app
+    ):
+        """A .py or .log is read by the plain-text fallthrough and must stay allowed."""
+        from application.api.user.attachments.routes import StoreAttachment
+
+        app = Flask(__name__)
+        mock_storage = MagicMock()
+        mock_storage.save_file.return_value = {"storage_type": "local"}
+        mock_store_attachment.return_value = SimpleNamespace(id="task-py")
+
+        with patch("application.api.user.base.storage", mock_storage), app.test_request_context(
+            "/api/store_attachment",
+            method="POST",
+            data={"file": (io.BytesIO(b"def main():\n    return 1\n"), "main.py")},
+            content_type="multipart/form-data",
+        ):
+            request.decoded_token = {"sub": "test_user"}
+            response = StoreAttachment().post()
+
+        assert _get_response_status(response) == 200
+        assert _get_response_json(response)["task_id"] == "task-py"
+        assert mock_store_attachment.call_count == 1
+
+    @patch("application.api.user.tasks.store_attachment.delay")
+    def test_batch_skips_unsupported_file_and_keeps_the_rest(
+        self, mock_store_attachment, flask_app
+    ):
+        from application.api.user.attachments.routes import StoreAttachment
+
+        app = Flask(__name__)
+        mock_storage = MagicMock()
+        mock_storage.save_file.return_value = {"storage_type": "local"}
+        mock_store_attachment.return_value = SimpleNamespace(id="task-notes")
+
+        with patch("application.api.user.base.storage", mock_storage), app.test_request_context(
+            "/api/store_attachment",
+            method="POST",
+            data={
+                "file": [
+                    (io.BytesIO(MP4_BYTES), "clip.mp4"),
+                    (io.BytesIO(b"hello"), "notes.txt"),
+                ]
+            },
+            content_type="multipart/form-data",
+        ):
+            request.decoded_token = {"sub": "test_user"}
+            response = StoreAttachment().post()
+
+        assert _get_response_status(response) == 200
+        body = _get_response_json(response)
+        assert [task["upload_index"] for task in body["tasks"]] == [1]
+        assert body["tasks"][0]["filename"] == "notes.txt"
+        assert body["errors"] == [
+            {"upload_index": 0, "filename": "clip.mp4", "error": "Unsupported file type: .mp4"}
+        ]
+        assert mock_storage.save_file.call_count == 1
+        assert mock_store_attachment.call_count == 1

tests/api/user/sources/test_wiki_routes.py

@@ -95,6 +95,12 @@ class TestCreateWikiSource:
         assert row["type"] == "wiki"
         assert row["config"]["kind"] == "wiki"
         assert int(row["tokens"]) == 0
+        # Wiki pages get embedded like any other source, so the row has to name
+        # the model that did it. NULL reads as "the legacy model" to the boot
+        # mismatch check, which then reports the source as stale forever.
+        from application.core.settings import settings
+
+        assert row["model"] == settings.EMBEDDINGS_NAME
         # No seed content → no re-embed, and never any ingest/reingest task.
         mock_reembed.assert_not_called()
         mock_ingest.assert_not_called()

tests/api/user/test_conversations.py

@@ -769,3 +769,88 @@ class TestSubmitFeedbackHappy:
             response = SubmitFeedback().post()
 
         assert response.status_code == 400
+
+
+@pytest.mark.unit
+class TestSubmitFeedbackWithApiKey:
+    """api_key callers carry no JWT."""
+
+    def _seed_agent_with_key(self, pg_conn, owner, key):
+        from application.storage.db.repositories.agents import AgentsRepository
+
+        return AgentsRepository(pg_conn).create(owner, "widget", "published", key=key)
+
+    def _post(self, app, pg_conn, payload):
+        from application.api.user.conversations.routes import SubmitFeedback
+
+        with _patch_conversations_db(pg_conn), app.test_request_context("/api/feedback", method="POST", json=payload):
+            from flask import request
+
+            request.decoded_token = None
+            return SubmitFeedback().post()
+
+    def test_valid_key_rates_its_own_conversation(self, app, pg_conn):
+        from application.storage.db.repositories.conversations import (
+            ConversationsRepository,
+        )
+
+        owner, key = "owner-fb-key", "agent-key-ok"
+        self._seed_agent_with_key(pg_conn, owner, key)
+        repo = ConversationsRepository(pg_conn)
+        conv_id = str(repo.create(owner, name="via widget", api_key=key)["id"])
+        repo.append_message(conv_id, {"prompt": "p", "response": "r"})
+
+        response = self._post(
+            app,
+            pg_conn,
+            {
+                "feedback": "LIKE",
+                "question_index": 0,
+                "conversation_id": conv_id,
+                "api_key": key,
+            },
+        )
+
+        assert response.status_code == 200
+        fb = repo.get_messages(conv_id)[0].get("feedback")
+        assert fb and fb.get("text") == "like"
+
+    def test_key_cannot_rate_owner_conversation_it_did_not_create(self, app, pg_conn):
+        from application.storage.db.repositories.conversations import (
+            ConversationsRepository,
+        )
+
+        owner, key = "owner-fb-scope", "agent-key-scope"
+        self._seed_agent_with_key(pg_conn, owner, key)
+        # Owner's conversation, not created with the key.
+        conv_id = _seed_conversation(pg_conn, owner, name="owner private")
+        ConversationsRepository(pg_conn).append_message(conv_id, {"prompt": "p", "response": "r"})
+
+        response = self._post(
+            app,
+            pg_conn,
+            {
+                "feedback": "LIKE",
+                "question_index": 0,
+                "conversation_id": conv_id,
+                "api_key": key,
+            },
+        )
+
+        assert response.status_code == 404
+
+    def test_unknown_key_is_unauthorized(self, app, pg_conn):
+        conv_id = _seed_conversation(pg_conn, "owner-fb-bad")
+
+        response = self._post(
+            app,
+            pg_conn,
+            {
+                "feedback": "LIKE",
+                "question_index": 0,
+                "conversation_id": conv_id,
+                "api_key": "no-such-key",
+            },
+        )
+
+        assert response.status_code == 401

tests/conftest.py

@@ -62,16 +62,18 @@ from sqlalchemy import create_engine
 _ALEMBIC_INI = Path(__file__).resolve().parent.parent / "application" / "alembic.ini"
 
 
-def _migrate_template_db(host, port, user, dbname, password, **_loader_options) -> None:
+def _migrate_template_db(host, port, user, dbname, password, **kwargs) -> None:
     """Run alembic ``upgrade head`` into the session's template database.
 
     Called once per session by pytest-postgresql's ``load=`` hook (against
     the template DB, before any test runs). Runs in a subprocess so the
     parent process never imports application settings with this URI cached.
 
-    ``**_loader_options`` absorbs keyword arguments newer pytest-postgresql
-    releases pass to loaders (9.0.0 added ``autocommit=``); alembic manages
-    its own connection, so they are irrelevant here.
+    ``**kwargs`` swallows keywords newer plugin releases hand to loaders --
+    9.0.0 added ``autocommit``, which broke every DB test on a plugin bump
+    alone. They describe the connection the caller already gave us, so
+    ignoring them keeps one signature working across versions instead of
+    pinning the plugin back.
     """
     url = (
         f"postgresql+psycopg://{user}:{password or ''}@{host}:{port}/{dbname}"
@@ -162,6 +164,21 @@ def _no_real_redis(monkeypatch):
     """
     monkeypatch.setattr("application.cache._redis_instance", None)
     monkeypatch.setattr("application.cache._redis_creation_failed", True)
+
+
+@pytest.fixture(autouse=True)
+def _no_worker_delegation(monkeypatch):
+    """Embed in-process during tests, the way CI has no worker to embed on.
+
+    ``EMBEDDINGS_DELEGATE_TO_WORKER`` ships on, so an unmocked embed would
+    publish to a broker nobody is consuming and block for
+    ``EMBEDDINGS_DELEGATE_TIMEOUT`` before failing -- a minute per call, and a
+    pass/fail that depends on whether the developer happens to have a worker
+    running. Tests covering delegation patch the setting back on themselves.
+    """
+    from application.core.settings import settings
+
+    monkeypatch.setattr(settings, "EMBEDDINGS_DELEGATE_TO_WORKER", False, raising=False)
     monkeypatch.setattr("application.cache._pubsub_redis_instance", None)
     monkeypatch.setattr("application.cache._pubsub_redis_creation_failed", True)
 

tests/graphrag/test_store.py

@@ -514,6 +514,27 @@ class TestEmbeddingDim:
         store = GraphStore.__new__(GraphStore)
         assert store._embedding_dim() == 1536
 
+    def test_none_dimension_falls_back_to_default(self, monkeypatch):
+        """A remote model outside the registry reports ``None``, not nothing.
+
+        ``getattr`` with a default cannot catch that -- the attribute exists --
+        so the width reached the DDL as ``vector(None)``.
+        """
+        from application.vectorstore import base as base_module
+
+        monkeypatch.setattr(base_module.settings, "EMBEDDINGS_BASE_URL", None)
+        fake_embedding = MagicMock()
+        fake_embedding.dimension = None
+        monkeypatch.setattr(
+            base_module.EmbeddingsSingleton,
+            "get_instance",
+            staticmethod(lambda *a, **k: fake_embedding),
+        )
+        monkeypatch.setattr(GraphStore, "_embedding_dim", _REAL_EMBEDDING_DIM)
+
+        store = GraphStore.__new__(GraphStore)
+        assert store._embedding_dim() == store_module.DEFAULT_NAME_EMBEDDING_DIM
+
     def test_falls_back_to_default_dimension(self, monkeypatch):
         from application.vectorstore import base as base_module
 

tests/llm/handlers/test_round_usage_persistence.py

@@ -179,8 +179,12 @@ class TestPerRoundUsageRows:
                 list(handler.handle_streaming(agent, first, {}, []))
 
         assert len(rows) == 2
-        assert rows[0] == {"prompt_tokens": 100, "generated_tokens": 10}
-        assert rows[1] == {"prompt_tokens": 200, "generated_tokens": 20}
+        # Provider-reported counts replace the estimates; the rest of the
+        # call record (e.g. ``model``) rides along, hence a subset check
+        # rather than dict equality.
+        counts = lambda row: {k: row[k] for k in ("prompt_tokens", "generated_tokens")}  # noqa: E731
+        assert counts(rows[0]) == {"prompt_tokens": 100, "generated_tokens": 10}
+        assert counts(rows[1]) == {"prompt_tokens": 200, "generated_tokens": 20}
 
 
 class TestPreferProviderUsageClaim:

tests/llm/test_base.py

@@ -2347,3 +2347,38 @@ class TestBuildConversationFromMessagesEmpty:
         result = handler._build_conversation_from_messages(messages)
         assert result is not None
         assert len(result.get("queries", [])) >= 1
+
+
+class TestFinishedLogCacheBins:
+    """``cached_tokens`` / ``cache_write_tokens`` ride on the finish events
+    only when a provider reported them, so dashboards can compute a cache
+    hit rate without a sentinel zero polluting providers that never report."""
+
+    def test_stream_finished_carries_cache_bins_when_given(self, caplog):
+        import logging as _logging
+
+        llm = StubLLM()
+        with caplog.at_level(_logging.INFO, logger="root"):
+            llm._emit_stream_finished_log(
+                "m1",
+                prompt_tokens=100,
+                completion_tokens=5,
+                latency_ms=10,
+                cached_tokens=80,
+                cache_write_tokens=7,
+            )
+        evt = next(r for r in caplog.records if r.message == "llm_stream_finished")
+        assert evt.cached_tokens == 80
+        assert evt.cache_write_tokens == 7
+
+    def test_gen_finished_omits_cache_bins_when_none(self, caplog):
+        import logging as _logging
+
+        llm = StubLLM()
+        with caplog.at_level(_logging.INFO, logger="root"):
+            llm._emit_gen_finished_log(
+                "m1", prompt_tokens=100, completion_tokens=5, latency_ms=10
+            )
+        evt = next(r for r in caplog.records if r.message == "llm_gen_finished")
+        assert not hasattr(evt, "cached_tokens")
+        assert not hasattr(evt, "cache_write_tokens")

tests/llm/test_openai_responses.py

@@ -1550,3 +1550,59 @@ def test_responses_gen_stream_retries_without_tools(monkeypatch):
     assert len(create.calls) == 2
     assert "tools" in create.calls[0]
     assert "tools" not in create.calls[1]
+
+
+@pytest.mark.unit
+def test_record_chat_usage_captures_cache_write_bin(monkeypatch):
+    """Newer OpenAI-family deployments report cache writes next to cache reads."""
+    llm = _make_llm(monkeypatch, ModelCapabilities(api_flavor="chat_completions"))
+    llm._record_chat_usage(_ns(
+        prompt_tokens=10,
+        completion_tokens=7,
+        total_tokens=17,
+        prompt_tokens_details=_ns(cached_tokens=4, cache_write_tokens=6),
+        completion_tokens_details=None,
+    ))
+    assert llm._last_usage["prompt_tokens_details"] == {
+        "cached_tokens": 4,
+        "cache_write_tokens": 6,
+    }
+
+
+@pytest.mark.unit
+def test_record_responses_metadata_captures_cache_write_bin(monkeypatch):
+    llm = _make_llm(monkeypatch, _responses_caps())
+    llm._record_responses_metadata(_ns(
+        id="resp_1",
+        usage=_ns(
+            input_tokens=10,
+            output_tokens=7,
+            total_tokens=17,
+            input_tokens_details=_ns(cached_tokens=0, cache_write_tokens=9),
+            output_tokens_details=None,
+        ),
+    ))
+    # A pure cache-write turn (first request of a prefix) has reads=0 and
+    # writes>0; both bins were reported, so both are recorded — a reported
+    # zero is "no cache hits", not "unknown".
+    assert llm._last_usage["prompt_tokens_details"] == {
+        "cached_tokens": 0,
+        "cache_write_tokens": 9,
+    }
+
+
+@pytest.mark.unit
+def test_record_responses_metadata_omits_unreported_cache_bins(monkeypatch):
+    """A provider that says nothing about caching must persist as NULL, not 0."""
+    llm = _make_llm(monkeypatch, _responses_caps())
+    llm._record_responses_metadata(_ns(
+        id="resp_2",
+        usage=_ns(
+            input_tokens=10,
+            output_tokens=7,
+            total_tokens=17,
+            input_tokens_details=None,
+            output_tokens_details=None,
+        ),
+    ))
+    assert "prompt_tokens_details" not in llm._last_usage

tests/parser/conftest.py

@@ -0,0 +1,23 @@
+import pytest
+
+
+@pytest.fixture(scope="session")
+def _mpnet_tokenizer():
+    """Real WordPiece tokenizer; skipped when the hub is unreachable."""
+    pytest.importorskip("tokenizers")
+    from tokenizers import Tokenizer
+
+    try:
+        tokenizer = Tokenizer.from_pretrained("sentence-transformers/all-mpnet-base-v2")
+    except Exception as exc:  # offline CI
+        pytest.skip(f"tokenizer unavailable: {exc}")
+    tokenizer.no_padding()
+    tokenizer.no_truncation()
+    return tokenizer
+
+
+@pytest.fixture
+def hf_counter(_mpnet_tokenizer):
+    from application.parser.tokenization import HuggingFaceCounter
+
+    return HuggingFaceCounter(_mpnet_tokenizer, "sentence-transformers/all-mpnet-base-v2")

tests/parser/file/test_constants.py

@@ -0,0 +1,109 @@
+"""Tests for the shared file-extension constants."""
+
+import re
+from importlib.util import find_spec
+from pathlib import Path
+
+import pytest
+
+from application.parser.file.bulk import get_default_file_extractor
+from application.parser.file.constants import (
+    ATTACHMENT_PARSER_EXTENSIONS,
+    attachment_extension,
+    has_attachment_parser,
+    SUPPORTED_SOURCE_EXTENSIONS,
+)
+
+FRONTEND_CONSTANTS = (
+    Path(__file__).resolve().parents[3] / "frontend/src/constants/fileUpload.ts"
+)
+
+
+@pytest.mark.unit
+def test_parser_extensions_match_the_extractor():
+    """The list is a hand-kept literal; this is what keeps it honest.
+
+    ``ATTACHMENT_PARSER_EXTENSIONS`` cannot import the extractor (that would
+    pull docling into the API process), so a parser added to
+    ``get_default_file_extractor`` would otherwise stay refused as an
+    attachment — which is how .webp, .tiff, .vtt and .xml were first missed.
+    A suffix listed here but *without* a parser is the opposite failure: it
+    skips the content check and reaches the plain-text fallthrough, which is
+    how a binary named notes.txt got through.
+    """
+    parser_suffixes = set(get_default_file_extractor())
+
+    assert parser_suffixes <= set(ATTACHMENT_PARSER_EXTENSIONS), (
+        "parsers exist for suffixes the attachment gate refuses: "
+        f"{sorted(parser_suffixes - set(ATTACHMENT_PARSER_EXTENSIONS))}"
+    )
+    if find_spec("docling") is not None:
+        assert parser_suffixes == set(ATTACHMENT_PARSER_EXTENSIONS), (
+            "listed as parser-backed but nothing parses them: "
+            f"{sorted(set(ATTACHMENT_PARSER_EXTENSIONS) - parser_suffixes)}"
+        )
+
+
+@pytest.mark.unit
+def test_frontend_mirrors_the_parser_extensions():
+    """The composer gates uploads client-side; a divergent list refuses valid files.
+
+    ``ATTACHMENT_PARSER_EXTENSIONS`` in the frontend is a hand-kept copy of
+    the backend's. Nothing but this test connects them, and .mdx, .xhtml and
+    .adoc were blocked in the UI while the API accepted them before it
+    existed.
+    """
+    if not FRONTEND_CONSTANTS.exists():
+        pytest.skip("frontend sources not present in this checkout")
+
+    source = FRONTEND_CONSTANTS.read_text(encoding="utf-8")
+    match = re.search(
+        r"ATTACHMENT_PARSER_EXTENSIONS:\s*readonly string\[\]\s*=\s*\[(.*?)\]",
+        source,
+        re.DOTALL,
+    )
+    assert match, "ATTACHMENT_PARSER_EXTENSIONS not found in fileUpload.ts"
+    frontend_extensions = set(re.findall(r"'(\.[^']+)'", match.group(1)))
+
+    assert frontend_extensions == set(ATTACHMENT_PARSER_EXTENSIONS), (
+        "frontend and backend attachment lists disagree — "
+        f"backend only: {sorted(set(ATTACHMENT_PARSER_EXTENSIONS) - frontend_extensions)}, "
+        f"frontend only: {sorted(frontend_extensions - set(ATTACHMENT_PARSER_EXTENSIONS))}"
+    )
+
+
+@pytest.mark.unit
+def test_source_ingestion_types_are_parser_backed_apart_from_text():
+    """Source ingestion's own list must not smuggle a parserless suffix past the sniff."""
+    assert set(SUPPORTED_SOURCE_EXTENSIONS) - set(ATTACHMENT_PARSER_EXTENSIONS) == {
+        ".txt"
+    }
+
+
+@pytest.mark.unit
+@pytest.mark.parametrize(
+    ("filename", "expected"),
+    [
+        ("Photo.JPG", ".jpg"),
+        ("/tmp/dir.d/report.PDF", ".pdf"),
+        ("archive.tar.gz", ".gz"),
+        ("Dockerfile", ""),
+        (".env", ""),
+        ("trailing.", "."),
+        (None, ""),
+    ],
+)
+def test_attachment_extension(filename, expected):
+    assert attachment_extension(filename) == expected
+
+
+@pytest.mark.unit
+def test_has_attachment_parser_is_case_insensitive_and_excludes_zip():
+    assert has_attachment_parser("Report.PDF")
+    assert has_attachment_parser("scan.WebP")
+    # No parser: admitted (or not) on content instead, never on the suffix.
+    assert not has_attachment_parser("archive.zip")
+    assert not has_attachment_parser("main.py")
+    assert not has_attachment_parser("clip.mp4")
+    # .txt is the plain-text fallthrough itself, not a parser.
+    assert not has_attachment_parser("notes.txt")

tests/parser/test_chunking.py

@@ -108,18 +108,89 @@ class TestSplitDocument:
 
         result = chunker.split_document(doc)
         assert len(result) > 1
-        # First chunk should contain header
         assert "h1" in result[0].text
+        # Only the first: this is what duplicate_headers=False means.
+        assert all("h1" not in chunk.text for chunk in result[1:])
 
     def test_split_duplicates_header(self):
+        """Every chunk carries the header when the flag is set.
+
+        Asserting only the first chunk passed even while the flag did nothing:
+        the implementation cleared the header after the first iteration, so
+        duplicate_headers was unreachable.
+        """
         chunker = Chunker(max_tokens=50, min_tokens=5, duplicate_headers=True)
         text = "h1\nh2\nh3\n" + "word " * 200
         doc = Document(text=text, doc_id="doc1")
 
         result = chunker.split_document(doc)
         assert len(result) > 1
-        # First chunk should contain header
-        assert "h1" in result[0].text
+        assert all("h1" in chunk.text for chunk in result)
+
+    def test_oversized_header_does_not_multiply_chunks(self):
+        """A header past the budget used to leave a one-token body budget.
+
+        ``max(1, max_tokens - header_tokens)`` bottomed out at 1, so the body
+        was cut into one chunk per token -- each still over ``max_tokens``,
+        since the whole header was prepended to it.
+        """
+        chunker = Chunker(max_tokens=50, min_tokens=5, duplicate_headers=True)
+        header = ("verylongheaderword " * 40) + "h2\nh3\n"
+        body = "word " * 100
+        doc = Document(text=f"{header}\n{body}", doc_id="doc1")
+
+        result = chunker.split_document(doc)
+
+        body_tokens = chunker.counter.count(body)
+        assert len(result) <= body_tokens // 10, "chunk count must track the budget"
+        for chunk in result:
+            assert chunk.extra_info["token_count"] <= chunker.max_tokens
+        assert "".join(chunk.text for chunk in result) == doc.text
+
+    def test_header_taking_most_of_the_budget_is_not_duplicated(self):
+        """Duplication is dropped when it would leave almost no room for body.
+
+        Below a quarter of the budget every chunk is mostly repeated header,
+        which multiplies the chunk count without adding retrievable text.
+        """
+        chunker = Chunker(max_tokens=50, min_tokens=5, duplicate_headers=True)
+        header = ("headerword " * 20) + "h2\nh3\n"
+        doc = Document(text=f"{header}\n" + "word " * 200, doc_id="doc1")
+
+        result = chunker.split_document(doc)
+
+        assert len(result) > 1
+        assert "headerword" in result[0].text
+        assert all("headerword" not in chunk.text for chunk in result[1:])
+
+    def test_header_only_document_is_not_dropped(self):
+        """The loop only ever emitted the header attached to a body piece.
+
+        With no body there was no piece, so an entire document disappeared
+        from the index with no error and no log line.
+        """
+        chunker = Chunker(max_tokens=50, min_tokens=1, duplicate_headers=False)
+        text = "h1\nh2\nh3\n"
+        doc = Document(text=text, doc_id="doc1")
+
+        result = chunker.split_document(doc)
+
+        assert len(result) == 1
+        assert result[0].text == text
+
+    def test_header_only_document_keeps_its_text_through_chunk(self):
+        """The reachable shape: three lines that alone exceed the budget."""
+        chunker = Chunker(max_tokens=5, min_tokens=1, duplicate_headers=False)
+        text = "h1\nh2\nh3\n"
+
+        result = chunker.chunk([Document(text=text, doc_id="doc1")])
+
+        assert result, "the document must not vanish"
+        assert "".join(chunk.text for chunk in result) == text
+
+    def test_zero_max_tokens_does_not_split_per_token(self):
+        chunker = Chunker(max_tokens=0, min_tokens=1)
+        assert chunker.max_tokens == 1
 
     def test_split_preserves_embedding(self):
         chunker = Chunker(max_tokens=50, min_tokens=5)

tests/parser/test_chunking_strategies.py

@@ -15,11 +15,13 @@ from application.parser.chunking_strategies import (
     SemanticChunker,
 )
 from application.parser.schema.base import Document
-from application.utils import get_encoding
+from application.parser.tokenization import get_token_counter
 
 
 def _tok(text: str) -> int:
-    return len(get_encoding().encode(text))
+    # Chunk budgets are enforced in the embedding model's tokenizer, so a
+    # test that measures them in cl100k measures the wrong thing.
+    return get_token_counter().count(text)
 
 
 @pytest.mark.unit

tests/parser/test_tokenization.py

@@ -0,0 +1,300 @@
+"""Chunk sizes must be counted in the embedding model's units, and splitting
+must never rewrite the text it splits."""
+
+import pytest
+
+from application.parser import tokenization
+from application.parser.tokenization import (
+    HuggingFaceCounter,
+    TiktokenCounter,
+    get_token_counter,
+)
+
+SAMPLES = [
+    "Hello World: DocsGPT ANSWERS Questions.",
+    "The quick brown fox jumps over the lazy dog. " * 40,
+    "Comment configurer l'authentification avec une clé API ?",
+    "def embed(text: str) -> list[float]:\n    return model.encode(text)\n",
+    "Ünïcödé — em-dashes, curly “quotes”, and 日本語 text.",
+    "a,b,c\n1,2,3\n4,5,6\n" * 30,
+]
+
+
+class _StubEncoding:
+    """Whitespace tokenizer standing in for tiktoken."""
+
+    def encode_ordinary(self, text):
+        return [ord(c) for c in text]
+
+    def decode(self, ids):
+        return "".join(chr(i) for i in ids)
+
+    def decode_with_offsets(self, ids):
+        # One token per character, so each token starts where the last ended.
+        return self.decode(ids), list(range(len(ids)))
+
+
+@pytest.fixture(autouse=True)
+def _clear_cache():
+    tokenization.reset_cache()
+    yield
+    tokenization.reset_cache()
+
+
+class TestSplittingPreservesText:
+    """The property that protects every stored document."""
+
+    @pytest.mark.parametrize("text", SAMPLES)
+    def test_tiktoken_split_reassembles_exactly(self, text, monkeypatch):
+        monkeypatch.setattr(tokenization, "get_encoding", _StubEncoding)
+        counter = TiktokenCounter()
+        pieces = counter.split(text, 7)
+        assert "".join(pieces) == text
+
+    @pytest.mark.parametrize("text", SAMPLES)
+    def test_hf_split_reassembles_exactly(self, text, hf_counter):
+        """WordPiece lowercases on decode, so splitting must slice, not decode."""
+        pieces = hf_counter.split(text, 7)
+        assert "".join(pieces) == text
+
+    def test_hf_split_does_not_lowercase(self, hf_counter):
+        text = "Hello World: DocsGPT ANSWERS Questions."
+        assert "".join(hf_counter.split(text, 3)) == text
+        assert "DocsGPT" in "".join(hf_counter.split(text, 3))
+
+    @pytest.mark.parametrize("text", SAMPLES)
+    def test_every_piece_is_within_budget(self, text, hf_counter):
+        budget = 10
+        for piece in hf_counter.split(text, budget):
+            # The final piece can absorb trailing characters the tokenizer
+            # dropped, so allow a small overshoot there only.
+            assert hf_counter.count(piece) <= budget + 2
+
+    def test_short_text_is_returned_whole(self, hf_counter):
+        assert hf_counter.split("short", 100) == ["short"]
+
+    def test_empty_text_yields_no_pieces(self, hf_counter):
+        assert hf_counter.split("", 10) == []
+
+    def test_zero_budget_is_clamped_not_infinite_loop(self, hf_counter):
+        pieces = hf_counter.split("some words here to split", 0)
+        assert "".join(pieces) == "some words here to split"
+
+
+class TestCounting:
+    def test_counts_differ_between_tokenizers(self, hf_counter, monkeypatch):
+        """The whole point: mpnet and cl100k disagree, so units matter."""
+        monkeypatch.setattr(tokenization, "get_encoding", _StubEncoding)
+        text = "internationalisation tokenization"
+        assert hf_counter.count(text) != TiktokenCounter().count(text)
+
+    def test_empty_text_counts_zero(self, hf_counter):
+        assert hf_counter.count("") == 0
+
+
+class TestSelection:
+    def test_registered_model_uses_its_own_tokenizer(self):
+        counter = get_token_counter("huggingface_sentence-transformers/all-mpnet-base-v2")
+        assert isinstance(counter, HuggingFaceCounter)
+        assert counter.name == "sentence-transformers/all-mpnet-base-v2"
+
+    def test_openai_model_falls_back_to_cl100k(self):
+        """OpenAI models are served remotely and genuinely count cl100k."""
+        assert isinstance(get_token_counter("openai_text-embedding-ada-002"), TiktokenCounter)
+
+    def test_unreachable_tokenizer_falls_back_rather_than_raising(self, monkeypatch):
+        monkeypatch.setattr(tokenization, "_load_hf_counter", lambda repo: None)
+        assert isinstance(get_token_counter("granite-311m"), TiktokenCounter)
+
+    def test_counter_is_cached_per_model(self):
+        first = get_token_counter("granite-311m")
+        assert get_token_counter("granite-311m") is first
+
+
+class TestTiktokenSplitAgainstRealCl100k:
+    """The stub above is one token per character, so it can never place a cut
+    inside a character. Real cl100k can, and that is the case that corrupted
+    text: decoding each window on its own turns a straddled multi-byte
+    character into U+FFFD on both sides of the cut."""
+
+    @pytest.fixture
+    def real_counter(self):
+        try:
+            counter = TiktokenCounter()
+            counter.count("probe")
+        except Exception as exc:  # offline CI, same policy as the HF fixture
+            pytest.skip(f"cl100k encoding unavailable: {exc}")
+        return counter
+
+    # 2000 is the shipped default max_tokens, 384 mpnet's window; the small
+    # values place many more cuts per unit of text.
+    @pytest.mark.parametrize("window", [1, 2, 3, 7, 128, 384, 2000])
+    @pytest.mark.parametrize(
+        "text",
+        [
+            "日本語のテキストです。絵文字も🎉あります。",
+            "検索は自然言語でできます。" * 200,
+            "Здравствуйте, как настроить аутентификацию?",
+            "🎉🎊✨🚀🔥💡📚🧠" * 50,
+            "Ünïcödé — em-dashes, curly “quotes”, and 日本語 text.",
+        ],
+        ids=["ja-short", "ja-long", "ru", "emoji", "mixed"],
+    )
+    def test_split_reassembles_exactly(self, real_counter, text, window):
+        pieces = real_counter.split(text, window)
+        assert "".join(pieces) == text
+
+    @pytest.mark.parametrize("window", [1, 3, 128, 2000])
+    def test_split_never_emits_a_replacement_character(self, real_counter, window):
+        text = "検索は自然言語でできます。絵文字も🎉あります。" * 100
+        assert "�" not in "".join(real_counter.split(text, window))
+
+    def test_first_window_budget_is_honoured_and_lossless(self, real_counter):
+        text = "日本語のテキストです。" * 50
+        pieces = real_counter.split(text, 20, first_max_tokens=5)
+        assert "".join(pieces) == text
+        assert real_counter.count(pieces[0]) <= 5
+
+
+class TestTiktokenCounterEdges:
+    """The cl100k path is the fallback, so its edges matter as much."""
+
+    @pytest.fixture
+    def counter(self, monkeypatch):
+        monkeypatch.setattr(tokenization, "get_encoding", _StubEncoding)
+        return TiktokenCounter()
+
+    def test_empty_text_counts_zero_and_splits_to_nothing(self, counter):
+        assert counter.count("") == 0
+        assert counter.split("", 10) == []
+
+    def test_text_within_budget_is_returned_whole(self, counter):
+        assert counter.split("abc", 10) == ["abc"]
+
+    def test_first_window_can_be_smaller_than_the_rest(self, counter):
+        """A header eats into the first chunk's budget only."""
+        pieces = counter.split("abcdefghij", 4, first_max_tokens=2)
+        assert pieces[0] == "ab"
+        assert "".join(pieces) == "abcdefghij"
+
+
+class TestCounterContract:
+    def test_base_class_requires_an_implementation(self):
+        base = tokenization.TokenCounter()
+        with pytest.raises(NotImplementedError):
+            base.count("x")
+        with pytest.raises(NotImplementedError):
+            base.split("x", 1)
+
+
+class TestFallbackWhenTokenizerUnavailable:
+    def test_load_failure_returns_none_rather_than_raising(self, monkeypatch, caplog):
+        """Chunking must survive an offline host or a bad repo name."""
+        import builtins
+
+        real_import = builtins.__import__
+
+        def boom(name, *args, **kwargs):
+            if name == "tokenizers":
+                raise ImportError("no tokenizers here")
+            return real_import(name, *args, **kwargs)
+
+        monkeypatch.setattr(builtins, "__import__", boom)
+        assert tokenization._load_hf_counter("some/repo") is None
+
+    def test_selection_falls_back_to_cl100k_on_failure(self, monkeypatch):
+        monkeypatch.setattr(tokenization, "_load_hf_counter", lambda repo: None)
+        assert isinstance(get_token_counter("granite-97m"), TiktokenCounter)
+
+    def test_reset_cache_forces_reselection(self, monkeypatch):
+        first = get_token_counter("granite-311m")
+        tokenization.reset_cache()
+        monkeypatch.setattr(tokenization, "_load_hf_counter", lambda repo: None)
+        assert get_token_counter("granite-311m") is not first
+
+
+class TestOffsetsWithoutSpans:
+    """Some tokenizers emit ``(0, 0)`` for specials or normalised-away chars.
+
+    Those tokens consume budget but point at no text, so the splitter has to
+    skip the window rather than emit an empty piece or lose the tail.
+    """
+
+    class _Encoded:
+        def __init__(self, offsets):
+            self.offsets = offsets
+            self.ids = list(range(len(offsets)))
+
+    class _Tokenizer:
+        def __init__(self, offsets):
+            self._offsets = offsets
+
+        def encode(self, text, add_special_tokens=False):
+            return TestOffsetsWithoutSpans._Encoded(self._offsets)
+
+    def _counter(self, offsets):
+        return HuggingFaceCounter(self._Tokenizer(offsets), "stub")
+
+    def test_span_less_windows_are_skipped_not_emitted_empty(self):
+        # Two real tokens, then a window of pure (0, 0) padding-like entries.
+        counter = self._counter([(0, 2), (2, 4), (0, 0), (0, 0)])
+        pieces = counter.split("abcd", 2)
+        assert "" not in pieces
+        assert "".join(pieces) == "abcd"
+
+    def test_trailing_text_is_never_dropped(self):
+        """Offsets that stop short of the string must not lose the remainder."""
+        counter = self._counter([(0, 1), (1, 2), (2, 3)])
+        pieces = counter.split("abcdef", 2)
+        assert "".join(pieces) == "abcdef"
+
+    def test_all_span_less_offsets_still_return_the_text(self):
+        counter = self._counter([(0, 0), (0, 0), (0, 0)])
+        assert "".join(counter.split("abc", 1)) == "abc"
+
+
+class TestUnknownTokenCollapse:
+    """A tokenizer that folds a long unbroken run into one ``[UNK]``.
+
+    WordPiece gives up on any word longer than ``max_input_chars_per_word``
+    and emits a single unknown token for it. Counting that as one token makes
+    a base64 blob or a minified bundle look tiny, so the chunker never splits
+    it and an oversized chunk reaches the embedding server.
+    """
+
+    class _CollapsingEncoding:
+        """One token per whitespace-separated word, however long the word."""
+
+        def __init__(self, text):
+            self.ids = []
+            self.offsets = []
+            cursor = 0
+            for word in text.split(" "):
+                if word:
+                    self.ids.append(0)
+                    self.offsets.append((cursor, cursor + len(word)))
+                cursor += len(word) + 1
+
+    class _CollapsingTokenizer:
+        def encode(self, text, add_special_tokens=False):
+            return TestUnknownTokenCollapse._CollapsingEncoding(text)
+
+    def _counter(self):
+        return tokenization.HuggingFaceCounter(self._CollapsingTokenizer(), "stub")
+
+    def test_long_unbroken_run_is_charged_by_its_span(self):
+        counter = self._counter()
+        assert counter.count("a" * 9000) > 100
+
+    def test_ordinary_prose_is_unaffected(self):
+        counter = self._counter()
+        text = "the quick brown fox jumps over the lazy dog"
+        assert counter.count(text) == 9
+
+    def test_split_bounds_a_collapsed_run(self):
+        counter = self._counter()
+        text = "a" * 9000
+        pieces = counter.split(text, 20)
+        assert "".join(pieces) == text, "split must not lose or alter text"
+        assert len(pieces) > 1, "a collapsed run must still be cut into pieces"
+        assert all(counter.count(p) <= 20 for p in pieces)

tests/scripts/test_prefetch_models.py

@@ -0,0 +1,96 @@
+"""Model pre-fetching, which the Docker build runs to bake artifacts in."""
+
+import sys
+import types
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from application.scripts import prefetch_models
+from application.vectorstore.model_registry import GRANITE_311M, MPNET, OPENAI_ADA_002
+
+
+@pytest.fixture
+def fake_fastembed():
+    """Stand in for FastEmbed so nothing is downloaded."""
+    text_embedding = MagicMock()
+    pooling = types.SimpleNamespace(CLS="CLS", MEAN="MEAN")
+    module = types.ModuleType("fastembed")
+    module.TextEmbedding = text_embedding
+    desc = types.ModuleType("fastembed.common.model_description")
+    desc.PoolingType = pooling
+    desc.ModelSource = lambda hf=None: {"hf": hf}
+    with patch.dict(
+        sys.modules,
+        {"fastembed": module, "fastembed.common.model_description": desc},
+    ):
+        yield text_embedding
+
+
+class TestPrefetch:
+    def test_defaults_cover_legacy_and_new_install(self):
+        """An upgraded image must still serve mpnet; a new one needs granite."""
+        assert MPNET.name in prefetch_models.DEFAULT_MODELS
+        assert GRANITE_311M.name in prefetch_models.DEFAULT_MODELS
+
+    def test_fetches_named_models(self, fake_fastembed):
+        fetched = prefetch_models.prefetch([GRANITE_311M.name])
+        assert fetched == [GRANITE_311M.repo]
+        assert fake_fastembed.call_args.kwargs["model_name"] == GRANITE_311M.repo
+
+    def test_registers_with_registry_pooling_and_dim(self, fake_fastembed):
+        prefetch_models.prefetch([GRANITE_311M.name])
+        kwargs = fake_fastembed.add_custom_model.call_args.kwargs
+        assert kwargs["model"] == GRANITE_311M.repo
+        assert kwargs["pooling"] == "CLS"
+        assert kwargs["dim"] == GRANITE_311M.dimension
+        assert kwargs["model_file"] == GRANITE_311M.onnx_file
+
+    def test_mean_pooled_model_registers_as_mean(self, fake_fastembed):
+        prefetch_models.prefetch([MPNET.name])
+        assert fake_fastembed.add_custom_model.call_args.kwargs["pooling"] == "MEAN"
+
+    def test_aliases_resolve(self, fake_fastembed):
+        assert prefetch_models.prefetch(["granite-311m"]) == [GRANITE_311M.repo]
+
+    def test_cache_dir_is_forwarded(self, fake_fastembed):
+        prefetch_models.prefetch([MPNET.name], cache_dir="/models")
+        assert fake_fastembed.call_args.kwargs["cache_dir"] == "/models"
+
+    def test_cache_dir_omitted_when_absent(self, fake_fastembed):
+        prefetch_models.prefetch([MPNET.name])
+        assert "cache_dir" not in fake_fastembed.call_args.kwargs
+
+    def test_remote_only_model_is_skipped(self, fake_fastembed):
+        """OpenAI embeddings have no local artifacts to cache."""
+        assert prefetch_models.prefetch([OPENAI_ADA_002.name]) == []
+        fake_fastembed.assert_not_called()
+
+    def test_unknown_model_fails_loudly(self, fake_fastembed):
+        """A silent skip at build time is a download at run time, offline."""
+        with pytest.raises(SystemExit) as excinfo:
+            prefetch_models.prefetch(["nope/nope"])
+        assert "nope/nope" in str(excinfo.value)
+        assert MPNET.name in str(excinfo.value)
+
+    def test_several_models_in_one_run(self, fake_fastembed):
+        fetched = prefetch_models.prefetch([MPNET.name, GRANITE_311M.name])
+        assert fetched == [MPNET.repo, GRANITE_311M.repo]
+
+
+class TestMain:
+    def test_no_args_fetches_the_defaults(self, fake_fastembed):
+        with patch.object(prefetch_models, "prefetch", return_value=[]) as spy:
+            assert prefetch_models.main([]) == 0
+        assert spy.call_args.args[0] == list(prefetch_models.DEFAULT_MODELS)
+
+    def test_explicit_args_override_the_defaults(self, fake_fastembed):
+        with patch.object(prefetch_models, "prefetch", return_value=[]) as spy:
+            prefetch_models.main(["granite-97m"])
+        assert spy.call_args.args[0] == ["granite-97m"]
+
+    def test_cache_dir_read_from_environment(self, fake_fastembed, monkeypatch):
+        monkeypatch.setenv("EMBEDDINGS_CACHE_DIR", "/app/models")
+        with patch.object(prefetch_models, "prefetch", return_value=[]) as spy:
+            prefetch_models.main(["granite-97m"])
+        assert spy.call_args.args[1] == "/app/models"

tests/scripts/test_reembed.py

@@ -0,0 +1,534 @@
+"""Re-embed script: CLI contract, orchestration, and the FAISS rebuild path."""
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from application.scripts import reembed
+
+
+def paginating_cursor(chunk_rows, *, graph_rows=(), graph_table=("graph_nodes",)):
+    """A cursor answering the reads ``reembed_pgvector`` issues, from memory.
+
+    The chunk read is paginated by keyset, so a cursor that returns the same
+    page for every ``fetchall`` never terminates. Returns the cursor and the
+    mutable table backing it.
+
+    Args:
+        chunk_rows: ``(id, text)`` rows for the source's chunk table.
+        graph_rows: ``(id, name)`` rows for ``graph_nodes``.
+        graph_table: What ``to_regclass`` reports; ``(None,)`` for absent.
+
+    Returns:
+        ``(cursor, tables)``, where ``tables["chunks"]`` and
+        ``tables["graph"]`` can be reassigned to change what is served.
+    """
+    tables = {"chunks": list(chunk_rows), "graph": list(graph_rows)}
+    pending = {"rows": []}
+    cursor = MagicMock()
+
+    def execute(query, params=None):
+        text = str(query)
+        if "to_regclass" in text:
+            pending["rows"] = [graph_table]
+        elif "FROM graph_nodes" in text:
+            pending["rows"] = list(tables["graph"])
+        elif "count(*)" in text:
+            pending["rows"] = [(len(tables["chunks"]),)]
+        elif "id > %s" in text:
+            _, after_id, limit = params
+            pending["rows"] = [r for r in tables["chunks"] if r[0] > after_id][:limit]
+        else:
+            pending["rows"] = list(tables["chunks"])[: params[1]]
+
+    cursor.execute.side_effect = execute
+    cursor.fetchall.side_effect = lambda: pending["rows"]
+    cursor.fetchone.side_effect = lambda: pending["rows"][0] if pending["rows"] else None
+    return cursor, tables
+
+
+class TestCLI:
+    def test_defaults(self):
+        args = reembed.build_parser().parse_args([])
+        assert args.dry_run is False
+        assert args.sources is None
+        assert args.batch_size == reembed.DEFAULT_BATCH_SIZE
+
+    def test_unsupported_store_exits_without_touching_anything(self):
+        with patch.object(reembed.settings, "VECTOR_STORE", "qdrant", create=True):
+            with patch.object(reembed, "run") as run:
+                assert reembed.main([]) == 2
+        run.assert_not_called()
+
+    def test_supported_stores_are_pgvector_and_faiss(self):
+        assert set(reembed.SUPPORTED_STORES) == {"pgvector", "faiss"}
+
+    def test_sources_are_split_and_trimmed(self):
+        with patch.object(reembed.settings, "VECTOR_STORE", "faiss", create=True):
+            with patch.object(reembed, "run", return_value=0) as run:
+                reembed.main(["--sources", " a , b ,, c "])
+        assert run.call_args.args[1] == ["a", "b", "c"]
+
+    def test_batch_size_is_clamped_to_at_least_one(self):
+        with patch.object(reembed.settings, "VECTOR_STORE", "faiss", create=True):
+            with patch.object(reembed, "run", return_value=0) as run:
+                reembed.main(["--batch-size", "0"])
+        assert run.call_args.args[2] == 1
+
+
+class TestRun:
+    def test_no_sources_is_a_clean_exit(self):
+        with patch.object(reembed, "list_source_ids", return_value=[]):
+            assert reembed.run("faiss", None, 8, False) == 0
+
+    def test_processes_every_discovered_source(self):
+        with patch.object(reembed, "list_source_ids", return_value=["a", "b"]):
+            with patch.object(reembed, "reembed_faiss", return_value=(3, 3)) as handler:
+                assert reembed.run("faiss", None, 8, False) == 0
+        assert [call.args[0] for call in handler.call_args_list] == ["a", "b"]
+
+    def test_explicit_sources_skip_discovery(self):
+        with patch.object(reembed, "list_source_ids") as discover:
+            with patch.object(reembed, "reembed_faiss", return_value=(1, 1)):
+                reembed.run("faiss", ["only-this"], 8, False)
+        discover.assert_not_called()
+
+    def test_one_failing_source_does_not_stop_the_others(self):
+        def handler(source_id, batch_size, dry_run):
+            if source_id == "bad":
+                raise RuntimeError("boom")
+            return (1, 1)
+
+        with patch.object(reembed, "reembed_faiss", side_effect=handler) as spy:
+            code = reembed.run("faiss", ["good", "bad", "also-good"], 8, False)
+        assert code == 1, "a failure must be reported in the exit code"
+        assert spy.call_count == 3, "later sources must still be attempted"
+
+    def test_dry_run_is_reported_as_success(self):
+        with patch.object(reembed, "reembed_faiss", return_value=(5, 0)):
+            assert reembed.run("faiss", ["a"], 8, True) == 0
+
+    def test_pgvector_uses_the_pgvector_handler(self):
+        with patch.object(reembed, "reembed_pgvector", return_value=(1, 1)) as handler:
+            reembed.run("pgvector", ["a"], 8, False)
+        handler.assert_called_once()
+
+
+class TestFaissRebuild:
+    @pytest.fixture
+    def stores(self):
+        """An existing store to read from and the rebuilt one written back."""
+        existing = MagicMock()
+        existing.get_chunks.return_value = [
+            {"doc_id": "1", "text": "alpha", "metadata": {"i": 0}},
+            {"doc_id": "2", "text": "beta", "metadata": {"i": 1}},
+        ]
+        rebuilt = MagicMock()
+        with patch.object(
+            reembed.VectorCreator, "create_vectorstore", side_effect=[existing, rebuilt]
+        ) as factory:
+            yield existing, rebuilt, factory
+
+    def test_rebuilds_from_stored_text_and_saves(self, stores):
+        existing, rebuilt, factory = stores
+        seen, written = reembed.reembed_faiss("s1", batch_size=8, dry_run=False)
+
+        assert (seen, written) == (2, 2)
+        rebuilt.save_local.assert_called_once()
+        docs = factory.call_args_list[1].kwargs["docs_init"]
+        assert [d.page_content for d in docs] == ["alpha", "beta"]
+        assert [d.metadata for d in docs] == [{"i": 0}, {"i": 1}]
+
+    def test_embeddings_key_comes_from_settings(self, stores):
+        """A placeholder here is sent as the server's bearer token."""
+        _, _, factory = stores
+        with patch.object(reembed.settings, "EMBEDDINGS_KEY", "sk-real", create=True):
+            reembed.reembed_faiss("s1", batch_size=8, dry_run=False)
+        keys = [call.kwargs.get("embeddings_key") for call in factory.call_args_list]
+        assert keys == ["sk-real", "sk-real"]
+
+    def test_chunk_ids_are_preserved(self, stores):
+        """Re-embedding must not renumber chunks.
+
+        Fresh ids orphan every GraphRAG ``graph_node_chunks`` row for the
+        source and invalidate any id a client already holds.
+        """
+        _, _, factory = stores
+        reembed.reembed_faiss("s1", batch_size=8, dry_run=False)
+        assert factory.call_args_list[1].kwargs["ids"] == ["1", "2"]
+
+    def test_existing_index_is_opened_past_the_dimension_check(self, stores):
+        """A width change is the main reason to run this script.
+
+        ``assert_embedding_dimensions`` refuses to open an index whose width
+        differs from the configured model -- and its error message recommends
+        this script, so without the opt-out the advice failed on every source.
+        The chunk text lives in the sidecar, so reading it needs no match.
+        """
+        _, _, factory = stores
+        reembed.reembed_faiss("s1", batch_size=8, dry_run=False)
+        assert factory.call_args_list[0].kwargs["skip_dimension_check"] is True
+        # The rebuild writes the new width, so it must still be checked.
+        assert "skip_dimension_check" not in factory.call_args_list[1].kwargs
+
+    def test_batch_size_is_forwarded_to_the_rebuild(self, stores):
+        """On a remote embeddings server the whole index is otherwise one POST."""
+        _, _, factory = stores
+        reembed.reembed_faiss("s1", batch_size=8, dry_run=False)
+        assert factory.call_args_list[1].kwargs["batch_size"] == 8
+
+    def test_existing_index_is_not_deleted(self, stores):
+        """The rebuild must not destroy the old index before the new one exists."""
+        existing, rebuilt, _ = stores
+        reembed.reembed_faiss("s1", batch_size=8, dry_run=False)
+        existing.delete_index.assert_not_called()
+
+    def test_dry_run_reads_but_never_rebuilds(self):
+        existing = MagicMock()
+        existing.get_chunks.return_value = [{"doc_id": "1", "text": "a", "metadata": {}}]
+        with patch.object(
+            reembed.VectorCreator, "create_vectorstore", return_value=existing
+        ) as factory:
+            seen, written = reembed.reembed_faiss("s1", batch_size=8, dry_run=True)
+        assert (seen, written) == (1, 0)
+        assert factory.call_count == 1, "no rebuild store may be constructed"
+
+    def test_empty_index_is_a_no_op(self):
+        existing = MagicMock()
+        existing.get_chunks.return_value = []
+        with patch.object(
+            reembed.VectorCreator, "create_vectorstore", return_value=existing
+        ):
+            assert reembed.reembed_faiss("s1", batch_size=8, dry_run=False) == (0, 0)
+
+
+class TestPgvectorWithoutTheExtension:
+    """Mocked pgvector paths.
+
+    The live tests in ``test_reembed_pgvector_live`` skip wherever the cluster
+    has no pgvector build -- which includes CI -- so the SQL shape and the
+    batching contract are pinned here too.
+    """
+
+    @pytest.fixture
+    def store(self):
+        """A store whose cursor serves the paginated reads, not a fixed page.
+
+        ``reembed_pgvector`` walks the source by keyset, so a cursor that
+        returns the same rows for every ``fetchall`` never terminates. The fake
+        answers each of the three queries the function issues from one in-memory
+        table, which tests mutate through ``rows``.
+        """
+        cursor, table = paginating_cursor([(1, "alpha"), (2, "beta"), (3, "gamma")])
+
+        store = MagicMock()
+        store._table_name = "documents"
+        store._vector_column = "embedding"
+        conn = MagicMock()
+        conn.cursor.return_value = cursor
+        store._get_connection.return_value = conn
+        store._embedding.embed_documents.side_effect = lambda texts: [
+            [0.5] * 4 for _ in texts
+        ]
+        with patch.object(
+            reembed.VectorCreator, "create_vectorstore", return_value=store
+        ), patch.object(reembed.settings, "GRAPHRAG_ENABLED", False):
+            yield store, conn, cursor, table
+
+    def test_reads_and_rewrites_every_chunk(self, store):
+        _, conn, cursor, _ = store
+        seen, written = reembed.reembed_pgvector("s1", batch_size=64, dry_run=False)
+        assert (seen, written) == (3, 3)
+        cursor.executemany.assert_called_once()
+        conn.commit.assert_called()
+
+    def test_pages_are_bounded_by_batch_size(self, store):
+        """The read is paginated, so a huge source never lands in memory at once."""
+        _, _, cursor, table = store
+        table["chunks"] = [(i, f"chunk-{i}") for i in range(1, 11)]
+        seen, written = reembed.reembed_pgvector("s1", batch_size=3, dry_run=False)
+        assert (seen, written) == (10, 10)
+        selects = [
+            call for call in cursor.execute.call_args_list
+            if "SELECT id, text" in str(call.args[0])
+        ]
+        # Four pages of at most 3, then one empty page to end the walk.
+        assert len(selects) == 5
+        assert all(call.args[1][-1] == 3 for call in selects)
+
+    def test_dry_run_counts_without_reading_the_text(self, store):
+        fake_store, _, cursor, _ = store
+        seen, written = reembed.reembed_pgvector("s1", batch_size=64, dry_run=True)
+        assert (seen, written) == (3, 0)
+        fake_store._embedding.embed_documents.assert_not_called()
+        cursor.executemany.assert_not_called()
+        assert any(
+            "count(*)" in str(call.args[0]) for call in cursor.execute.call_args_list
+        )
+
+    def test_batches_commit_separately(self, store):
+        _, conn, cursor, _ = store
+        reembed.reembed_pgvector("s1", batch_size=2, dry_run=False)
+        # 3 rows at batch 2 is two write transactions.
+        assert cursor.executemany.call_count == 2
+        assert conn.commit.call_count == 2
+
+    def test_failed_batch_rolls_back_and_raises(self, store):
+        _, conn, cursor, _ = store
+        cursor.executemany.side_effect = RuntimeError("write failed")
+        with pytest.raises(RuntimeError):
+            reembed.reembed_pgvector("s1", batch_size=64, dry_run=False)
+        conn.rollback.assert_called_once()
+
+    def test_connection_is_returned_even_on_failure(self, store):
+        fake_store, _, cursor, _ = store
+        cursor.executemany.side_effect = RuntimeError("boom")
+        with pytest.raises(RuntimeError):
+            reembed.reembed_pgvector("s1", batch_size=64, dry_run=False)
+        fake_store.close.assert_called_once()
+
+    def test_null_text_does_not_crash_the_embed_call(self, store):
+        fake_store, _, _, table = store
+        table["chunks"] = [(1, None), (2, "beta")]
+        seen, written = reembed.reembed_pgvector("s1", batch_size=64, dry_run=False)
+        assert (seen, written) == (2, 2)
+        assert fake_store._embedding.embed_documents.call_args.args[0] == ["", "beta"]
+
+    def test_empty_source_is_a_no_op(self, store):
+        fake_store, _, _, table = store
+        table["chunks"] = []
+        assert reembed.reembed_pgvector("s1", batch_size=64, dry_run=False) == (0, 0)
+        fake_store._embedding.embed_documents.assert_not_called()
+
+    def test_source_discovery_returns_sorted_ids(self):
+        store = MagicMock()
+        store._table_name = "documents"
+        cursor = MagicMock()
+        cursor.fetchall.return_value = [("b",), ("a",)]
+        conn = MagicMock()
+        conn.cursor.return_value = cursor
+        store._get_connection.return_value = conn
+        with patch.object(
+            reembed.VectorCreator, "create_vectorstore", return_value=store
+        ):
+            assert reembed.list_source_ids("pgvector") == ["b", "a"]
+
+
+class TestFaissSourceDiscovery:
+    def test_source_ids_come_from_index_directories(self):
+        storage = MagicMock()
+        storage.list_files.return_value = [
+            "indexes/src-a/index.faiss",
+            "indexes/src-a/index.pkl",
+            "indexes/src-b/index.faiss",
+        ]
+        with patch(
+            "application.storage.storage_creator.StorageCreator.get_storage",
+            return_value=storage,
+        ):
+            assert reembed.list_source_ids("faiss") == ["src-a", "src-b"]
+
+    def test_storage_failure_is_reported_as_a_usable_error(self):
+        storage = MagicMock()
+        storage.list_files.side_effect = OSError("permission denied")
+        with patch(
+            "application.storage.storage_creator.StorageCreator.get_storage",
+            return_value=storage,
+        ):
+            with pytest.raises(reembed.ReembedError, match="permission denied"):
+                reembed.list_source_ids("faiss")
+
+    def test_unsupported_store_error_names_the_alternatives(self):
+        with patch.object(reembed.settings, "VECTOR_STORE", "milvus", create=True):
+            assert reembed.main([]) == 2
+
+
+class TestGraphNodeReembedding:
+    """``graph_nodes.name_embedding`` seeds every graph traversal.
+
+    It is written once at extraction time and never revisited, so rewriting
+    only the chunk table leaves the graph seeding from the previous model --
+    and since mpnet and granite-311m are both 768-dimensional, the column
+    accepts the mismatch and nothing reports it.
+    """
+
+    @pytest.fixture
+    def graph(self):
+        store = MagicMock()
+        store._embedding.embed_documents.side_effect = lambda texts: [
+            [0.5] * 4 for _ in texts
+        ]
+        cursor = MagicMock()
+        cursor.fetchone.return_value = ("graph_nodes",)
+        cursor.fetchall.return_value = [
+            ("n1", "Alpha"),
+            ("n2", "Beta"),
+            ("n3", "Gamma"),
+        ]
+        conn = MagicMock()
+        conn.cursor.return_value = cursor
+        return store, conn, cursor
+
+    def test_rewrites_every_node_name(self, graph):
+        store, conn, cursor = graph
+        written = reembed.reembed_graph_nodes(
+            store, conn, "s1", batch_size=64, dry_run=False
+        )
+        assert written == 3
+        store._embedding.embed_documents.assert_called_once_with(
+            ["Alpha", "Beta", "Gamma"]
+        )
+        statement = cursor.executemany.call_args.args[0]
+        assert "graph_nodes" in statement and "name_embedding" in statement
+        conn.commit.assert_called()
+
+    def test_dry_run_counts_without_embedding(self, graph):
+        store, conn, cursor = graph
+        assert reembed.reembed_graph_nodes(store, conn, "s1", 64, dry_run=True) == 3
+        store._embedding.embed_documents.assert_not_called()
+        cursor.executemany.assert_not_called()
+
+    def test_missing_table_is_a_no_op(self, graph):
+        store, conn, cursor = graph
+        cursor.fetchone.return_value = (None,)
+        assert reembed.reembed_graph_nodes(store, conn, "s1", 64, dry_run=False) == 0
+        store._embedding.embed_documents.assert_not_called()
+
+    def test_batches_commit_separately(self, graph):
+        store, conn, cursor = graph
+        reembed.reembed_graph_nodes(store, conn, "s1", batch_size=2, dry_run=False)
+        assert cursor.executemany.call_count == 2
+        assert conn.commit.call_count == 2
+
+    def test_failed_batch_rolls_back_and_raises(self, graph):
+        store, conn, cursor = graph
+        cursor.executemany.side_effect = RuntimeError("write failed")
+        with pytest.raises(RuntimeError, match="write failed"):
+            reembed.reembed_graph_nodes(store, conn, "s1", 64, dry_run=False)
+        conn.rollback.assert_called_once()
+
+    def test_pgvector_run_skips_the_graph_when_disabled(self):
+        store, conn, cursor = self._pgvector_mocks()
+        with patch.object(
+            reembed.VectorCreator, "create_vectorstore", return_value=store
+        ), patch.object(reembed.settings, "GRAPHRAG_ENABLED", False):
+            reembed.reembed_pgvector("s1", batch_size=64, dry_run=False)
+        assert not self._graph_statements(cursor)
+
+    def test_pgvector_run_reembeds_the_graph_when_enabled(self):
+        store, conn, cursor = self._pgvector_mocks()
+        with patch.object(
+            reembed.VectorCreator, "create_vectorstore", return_value=store
+        ), patch.object(reembed.settings, "GRAPHRAG_ENABLED", True):
+            reembed.reembed_pgvector("s1", batch_size=64, dry_run=False)
+        assert self._graph_statements(cursor)
+
+    @staticmethod
+    def _pgvector_mocks():
+        cursor, _ = paginating_cursor(
+            [(1, "alpha"), (2, "beta")],
+            graph_rows=[("n1", "Alpha"), ("n2", "Beta")],
+        )
+        store = MagicMock()
+        store._table_name = "documents"
+        store._vector_column = "embedding"
+        conn = MagicMock()
+        conn.cursor.return_value = cursor
+        store._get_connection.return_value = conn
+        store._embedding.embed_documents.side_effect = lambda texts: [
+            [0.5] * 4 for _ in texts
+        ]
+        return store, conn, cursor
+
+    @staticmethod
+    def _graph_statements(cursor):
+        return [
+            call
+            for call in cursor.executemany.call_args_list
+            if "graph_nodes" in str(call.args[0])
+        ]
+
+
+class TestEmbedsInProcess:
+    """A batch job should not round-trip every chunk through the broker."""
+
+    def test_delegation_is_turned_off_for_the_run(self, monkeypatch):
+        from application.core.settings import settings
+
+        monkeypatch.setattr(settings, "EMBEDDINGS_DELEGATE_TO_WORKER", True, raising=False)
+        monkeypatch.setattr(settings, "VECTOR_STORE", "pgvector", raising=False)
+        seen = {}
+        with patch.object(reembed, "run", side_effect=lambda *a, **k: seen.setdefault(
+            "delegating", settings.EMBEDDINGS_DELEGATE_TO_WORKER
+        ) or 0):
+            assert reembed.main(["--dry-run"]) == 0
+        assert seen["delegating"] is False
+
+
+class TestRecordsTheModel:
+    """``sources.model`` is what the boot mismatch check reads."""
+
+    def test_a_re_embedded_source_is_stamped(self, monkeypatch):
+        from application.core.settings import settings
+
+        monkeypatch.setattr(settings, "EMBEDDINGS_NAME", "new/model", raising=False)
+        conn = MagicMock()
+        session = MagicMock()
+        session.__enter__ = MagicMock(return_value=conn)
+        session.__exit__ = MagicMock(return_value=False)
+        with patch("application.storage.db.session.db_session", return_value=session):
+            reembed.record_source_model("src-1")
+        params = conn.execute.call_args.args[1]
+        assert params == {"model": "new/model", "id": "src-1"}
+
+    def test_a_dry_run_stamps_nothing(self, monkeypatch):
+        with patch.object(reembed, "record_source_model") as record, patch.object(
+            reembed, "list_source_ids", return_value=["a"]
+        ), patch.object(reembed, "reembed_pgvector", return_value=(3, 0)):
+            reembed.run("pgvector", None, 64, True)
+        record.assert_not_called()
+
+    def test_a_real_run_stamps_each_source(self, monkeypatch):
+        with patch.object(reembed, "record_source_model") as record, patch.object(
+            reembed, "list_source_ids", return_value=["a", "b"]
+        ), patch.object(reembed, "reembed_pgvector", return_value=(3, 3)):
+            reembed.run("pgvector", None, 64, False)
+        assert [c.args[0] for c in record.call_args_list] == ["a", "b"]
+
+    def test_a_failed_source_is_not_stamped(self):
+        with patch.object(reembed, "record_source_model") as record, patch.object(
+            reembed, "list_source_ids", return_value=["a"]
+        ), patch.object(reembed, "reembed_pgvector", side_effect=RuntimeError("boom")):
+            assert reembed.run("pgvector", None, 64, False) == 1
+        record.assert_not_called()
+
+
+class TestThePinIsResolved:
+    """The script must embed with the model the installation is pinned to.
+
+    ``resolve_embeddings_pin`` runs in ``application.app``, which this script
+    never imports. An install pinned in ``app_metadata`` with no
+    ``EMBEDDINGS_NAME`` in the environment -- every stock Kubernetes
+    deployment, whose manifests carry no embedding config -- would otherwise
+    rewrite its whole index with the legacy code default and stamp
+    ``sources.model`` to match, creating the cross-model index this script
+    exists to repair.
+    """
+
+    def test_main_resolves_the_pin_before_reading_the_store(self):
+        order = []
+        with patch(
+            "application.storage.db.embeddings_pin.resolve_embeddings_pin",
+            side_effect=lambda *a, **k: order.append("pin"),
+        ), patch.object(reembed.settings, "VECTOR_STORE", "pgvector", create=True), patch.object(
+            reembed, "run", side_effect=lambda *a, **k: (order.append("run"), 0)[1]
+        ):
+            assert reembed.main([]) == 0
+        assert order == ["pin", "run"], "the pin must resolve before anything embeds"
+
+    def test_an_unsupported_store_still_resolved_the_pin_first(self):
+        with patch(
+            "application.storage.db.embeddings_pin.resolve_embeddings_pin"
+        ) as pin, patch.object(reembed.settings, "VECTOR_STORE", "qdrant", create=True):
+            assert reembed.main([]) == 2
+        pin.assert_called_once()

tests/scripts/test_reembed_pgvector_live.py

@@ -0,0 +1,297 @@
+"""Live pgvector run of the re-embed script.
+
+Uses the ephemeral pytest-postgresql cluster with a stub embeddings model, so
+a real ``UPDATE ... ::vector`` round trip is exercised without downloading a
+model. Skips when the cluster has no pgvector build.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import patch
+
+import pytest
+
+from application.scripts import reembed
+from application.vectorstore import pgvector as pgvector_module
+from application.vectorstore.pgvector import PGVectorStore
+
+pytestmark = pytest.mark.integration
+
+DIM = 8
+
+
+class _Embeddings:
+    """Returns a distinct constant per generation, so a rewrite is visible."""
+
+    dimension = DIM
+
+    def __init__(self, seed: float):
+        self.seed = seed
+        self.calls = 0
+
+    def embed_documents(self, texts):
+        self.calls += 1
+        return [[self.seed] + [0.0] * (DIM - 1) for _ in texts]
+
+    def embed_query(self, query):
+        return [self.seed] + [0.0] * (DIM - 1)
+
+
+def _dsn(info) -> str:
+    password = f":{info.password}" if info.password else ""
+    return f"postgresql://{info.user}{password}@{info.host}:{info.port}/{info.dbname}"
+
+
+@pytest.fixture(autouse=True)
+def _close_pools():
+    """Never leak a pool into another test; the DSN dies with the test DB."""
+    yield
+    for dsn, pool in list(pgvector_module._POOLS.items()):
+        try:
+            pool.close()
+        except Exception:
+            # Teardown only: the ephemeral cluster may already be gone, and a
+            # failure to close a pool for a dead DSN must not fail the test
+            # that just passed. Dropping the entry below is what matters.
+            pass
+        pgvector_module._POOLS.pop(dsn, None)
+
+
+@pytest.fixture
+def live_dsn(postgresql, monkeypatch):
+    try:
+        with postgresql.cursor() as cursor:
+            cursor.execute("CREATE EXTENSION vector;")
+        postgresql.rollback()
+    except Exception as exc:
+        postgresql.rollback()
+        pytest.skip(f"pgvector extension unavailable: {exc}")
+
+    dsn = _dsn(postgresql.info)
+    from application.core import settings as settings_module
+
+    settings = settings_module.settings
+    monkeypatch.setattr(settings, "VECTOR_STORE", "pgvector", raising=False)
+    monkeypatch.setattr(settings, "PGVECTOR_CONNECTION_STRING", dsn, raising=False)
+    monkeypatch.setattr(settings, "GRAPHRAG_ENABLED", False, raising=False)
+    monkeypatch.setattr(settings, "PGVECTOR_IVFFLAT_PROBES", None, raising=False)
+    monkeypatch.setattr(settings, "PGVECTOR_POOL_MAX_SIZE", 4, raising=False)
+    monkeypatch.setattr(settings, "EMBEDDINGS_NAME", "granite-311m", raising=False)
+    return dsn
+
+
+def _seed(dsn, source_id, texts, embeddings):
+    """Create the schema and insert ``texts`` embedded by ``embeddings``."""
+    with patch(
+        "application.vectorstore.base.BaseVectorStore._get_embeddings",
+        return_value=embeddings,
+    ):
+        store = PGVectorStore(source_id=source_id, connection_string=dsn)
+        conn = store._get_connection()
+        PGVectorStore.create_schema(conn, dimension=DIM)
+        conn.commit()
+        store.add_texts(list(texts), metadatas=[{"i": i} for i in range(len(texts))])
+        store.close()
+
+
+def _as_list(vector):
+    """Normalise a stored vector to a list of floats.
+
+    Depending on whether pgvector's adapter is registered on the reading
+    connection, the value comes back as a ``Vector`` or as its text form
+    ``'[1,0,...]'``. Both are valid; the assertions should not care.
+    """
+    if vector is None:
+        return []
+    if hasattr(vector, "to_list"):
+        return list(vector.to_list())
+    if isinstance(vector, str):
+        return [float(part) for part in vector.strip("[]").split(",") if part]
+    return list(vector)
+
+
+def _vectors(dsn, source_id):
+    store = PGVectorStore(source_id=source_id, connection_string=dsn)
+    conn = store._get_connection()
+    cursor = conn.cursor()
+    try:
+        cursor.execute(
+            "SELECT text, embedding FROM documents WHERE source_id = %s ORDER BY id",
+            (source_id,),
+        )
+        # pgvector hands back a ``Vector``; normalise to a plain list so the
+        # assertions read the same whichever adapter is registered.
+        return [(text, _as_list(vector)) for text, vector in cursor.fetchall()]
+    finally:
+        cursor.close()
+        store.close()
+
+
+TEXTS = ["alpha document", "beta document", "gamma document"]
+
+
+class TestReembedPgvectorLive:
+    def test_rewrites_vectors_and_preserves_text(self, live_dsn):
+        _seed(live_dsn, "src-a", TEXTS, _Embeddings(1.0))
+        before = _vectors(live_dsn, "src-a")
+        assert [row[0] for row in before] == TEXTS
+        assert all(row[1][0] == pytest.approx(1.0) for row in before)
+
+        new_model = _Embeddings(9.0)
+        with patch(
+            "application.vectorstore.base.BaseVectorStore._get_embeddings",
+            return_value=new_model,
+        ):
+            seen, written = reembed.reembed_pgvector("src-a", batch_size=2, dry_run=False)
+
+        assert (seen, written) == (3, 3)
+        after = _vectors(live_dsn, "src-a")
+        # Text is untouched; only the vectors moved.
+        assert [row[0] for row in after] == TEXTS
+        assert all(row[1][0] == pytest.approx(9.0) for row in after)
+
+    def test_dry_run_counts_without_writing(self, live_dsn):
+        _seed(live_dsn, "src-b", TEXTS, _Embeddings(1.0))
+        new_model = _Embeddings(9.0)
+        with patch(
+            "application.vectorstore.base.BaseVectorStore._get_embeddings",
+            return_value=new_model,
+        ):
+            seen, written = reembed.reembed_pgvector("src-b", batch_size=2, dry_run=True)
+
+        assert (seen, written) == (3, 0)
+        assert new_model.calls == 0, "dry run must not embed"
+        assert all(row[1][0] == pytest.approx(1.0) for row in _vectors(live_dsn, "src-b"))
+
+    def test_batches_are_respected(self, live_dsn):
+        _seed(live_dsn, "src-c", TEXTS, _Embeddings(1.0))
+        new_model = _Embeddings(9.0)
+        with patch(
+            "application.vectorstore.base.BaseVectorStore._get_embeddings",
+            return_value=new_model,
+        ):
+            reembed.reembed_pgvector("src-c", batch_size=2, dry_run=False)
+        assert new_model.calls == 2, "3 chunks at batch 2 is two embed calls"
+
+    def test_only_the_named_source_is_touched(self, live_dsn):
+        _seed(live_dsn, "src-d", TEXTS, _Embeddings(1.0))
+        _seed(live_dsn, "src-e", TEXTS, _Embeddings(1.0))
+        with patch(
+            "application.vectorstore.base.BaseVectorStore._get_embeddings",
+            return_value=_Embeddings(9.0),
+        ):
+            reembed.reembed_pgvector("src-d", batch_size=64, dry_run=False)
+
+        assert all(row[1][0] == pytest.approx(9.0) for row in _vectors(live_dsn, "src-d"))
+        assert all(row[1][0] == pytest.approx(1.0) for row in _vectors(live_dsn, "src-e"))
+
+    def test_source_discovery_lists_every_source(self, live_dsn):
+        _seed(live_dsn, "src-f", TEXTS, _Embeddings(1.0))
+        _seed(live_dsn, "src-g", TEXTS, _Embeddings(1.0))
+        assert reembed.list_source_ids("pgvector") == ["src-f", "src-g"]
+
+
+GRAPH_SOURCE = "11111111-2222-3333-4444-555555555555"
+
+
+def _seed_graph_node(dsn, source_id, name, seed):
+    """Insert one graph node carrying a name embedding at ``seed``."""
+    from application.graphrag.store import GraphStore
+
+    store = PGVectorStore(source_id=source_id, connection_string=dsn)
+    conn = store._get_connection()
+    GraphStore.create_schema(conn, dimension=DIM)
+    cursor = conn.cursor()
+    try:
+        cursor.execute(
+            "INSERT INTO graph_nodes (id, source_id, name, normalized_name, type, "
+            "description, degree, doc_freq, name_embedding) "
+            "VALUES (gen_random_uuid(), %s, %s, %s, 'ENTITY', '', 0, 0, %s::vector)",
+            (source_id, name, name.lower(), str([seed] + [0.0] * (DIM - 1))),
+        )
+        conn.commit()
+    finally:
+        cursor.close()
+        store.close()
+
+
+def _node_vectors(dsn, source_id):
+    store = PGVectorStore(source_id=source_id, connection_string=dsn)
+    conn = store._get_connection()
+    cursor = conn.cursor()
+    try:
+        cursor.execute(
+            "SELECT name, name_embedding FROM graph_nodes "
+            "WHERE source_id = %s ORDER BY name",
+            (source_id,),
+        )
+        return [(name, _as_list(vector)) for name, vector in cursor.fetchall()]
+    finally:
+        cursor.close()
+        store.close()
+
+
+class TestGraphNodeReembedLive:
+    """``graph_nodes.name_embedding`` seeds every graph traversal.
+
+    Rewriting only the chunk table leaves it in the previous model's space,
+    and because mpnet and granite-311m share a width the column accepts the
+    mismatch silently -- exactly the failure the script exists to prevent.
+    """
+
+    def test_node_names_are_re_embedded(self, live_dsn, monkeypatch):
+        from application.core import settings as settings_module
+
+        monkeypatch.setattr(
+            settings_module.settings, "GRAPHRAG_ENABLED", True, raising=False
+        )
+        _seed(live_dsn, GRAPH_SOURCE, TEXTS, _Embeddings(1.0))
+        _seed_graph_node(live_dsn, GRAPH_SOURCE, "Alpha", 1.0)
+
+        assert all(v[0] == pytest.approx(1.0) for _, v in _node_vectors(live_dsn, GRAPH_SOURCE))
+
+        with patch(
+            "application.vectorstore.base.BaseVectorStore._get_embeddings",
+            return_value=_Embeddings(9.0),
+        ):
+            reembed.reembed_pgvector(GRAPH_SOURCE, batch_size=64, dry_run=False)
+
+        after = _node_vectors(live_dsn, GRAPH_SOURCE)
+        assert [name for name, _ in after] == ["Alpha"]
+        assert all(v[0] == pytest.approx(9.0) for _, v in after), (
+            "graph node names must move with the chunk vectors"
+        )
+
+    def test_graph_is_left_alone_when_graphrag_is_off(self, live_dsn, monkeypatch):
+        from application.core import settings as settings_module
+
+        monkeypatch.setattr(
+            settings_module.settings, "GRAPHRAG_ENABLED", False, raising=False
+        )
+        _seed(live_dsn, GRAPH_SOURCE, TEXTS, _Embeddings(1.0))
+        _seed_graph_node(live_dsn, GRAPH_SOURCE, "Alpha", 1.0)
+
+        with patch(
+            "application.vectorstore.base.BaseVectorStore._get_embeddings",
+            return_value=_Embeddings(9.0),
+        ):
+            reembed.reembed_pgvector(GRAPH_SOURCE, batch_size=64, dry_run=False)
+
+        assert all(v[0] == pytest.approx(1.0) for _, v in _node_vectors(live_dsn, GRAPH_SOURCE))
+
+    def test_dry_run_leaves_node_vectors_untouched(self, live_dsn, monkeypatch):
+        from application.core import settings as settings_module
+
+        monkeypatch.setattr(
+            settings_module.settings, "GRAPHRAG_ENABLED", True, raising=False
+        )
+        _seed(live_dsn, GRAPH_SOURCE, TEXTS, _Embeddings(1.0))
+        _seed_graph_node(live_dsn, GRAPH_SOURCE, "Alpha", 1.0)
+
+        with patch(
+            "application.vectorstore.base.BaseVectorStore._get_embeddings",
+            return_value=_Embeddings(9.0),
+        ):
+            reembed.reembed_pgvector(GRAPH_SOURCE, batch_size=64, dry_run=True)
+
+        assert all(v[0] == pytest.approx(1.0) for _, v in _node_vectors(live_dsn, GRAPH_SOURCE))

tests/storage/db/repositories/test_token_usage.py

@@ -290,3 +290,32 @@ class TestBucketedTotals:
         )
         assert len(rows) == 1
         assert rows[0]["prompt_tokens"] == 10
+
+
+class TestCacheBins:
+    def _row(self, pg_conn, user_id):
+        return pg_conn.execute(
+            text(
+                "SELECT cached_tokens, cache_write_tokens FROM token_usage "
+                "WHERE user_id = :u ORDER BY id DESC LIMIT 1"
+            ),
+            {"u": user_id},
+        ).fetchone()
+
+    def test_persists_cache_bins(self, pg_conn):
+        repo = TokenUsageRepository(pg_conn)
+        repo.insert(
+            user_id="cache-user",
+            prompt_tokens=1000,
+            generated_tokens=10,
+            cached_tokens=800,
+            cache_write_tokens=100,
+        )
+        assert tuple(self._row(pg_conn, "cache-user")) == (800, 100)
+
+    def test_cache_bins_default_to_null_not_zero(self, pg_conn):
+        """NULL means "provider did not report"; 0 means "reported no cache
+        activity". Keeping them distinct is what makes a hit-rate query honest."""
+        repo = TokenUsageRepository(pg_conn)
+        repo.insert(user_id="cache-user-2", prompt_tokens=10, generated_tokens=1)
+        assert tuple(self._row(pg_conn, "cache-user-2")) == (None, None)

tests/storage/db/test_bootstrap_vector_schema.py

@@ -74,8 +74,8 @@ class TestEnsureVectorSchemaCreates:
         cursor = MagicMock()
         conn.cursor.return_value = cursor
         with patch("psycopg.connect", return_value=conn) as connect, patch(
-            "application.vectorstore.base.get_embeddings",
-            return_value=_embeddings(dimension),
+            "application.vectorstore.model_registry.dimension_for",
+            return_value=dimension,
         ), patch(
             "application.vectorstore.pgvector.PGVectorStore.create_schema"
         ) as vector_schema, patch(
@@ -130,8 +130,8 @@ class TestEnsureVectorSchemaDimensionCheck:
     ):
         conn = MagicMock()
         with patch("psycopg.connect", return_value=conn), patch(
-            "application.vectorstore.base.get_embeddings",
-            return_value=_embeddings(1536),
+            "application.vectorstore.model_registry.dimension_for",
+            return_value=1536,
         ), patch("application.vectorstore.pgvector.PGVectorStore.create_schema"), patch(
             "application.vectorstore.pgvector.PGVectorStore.table_dimension",
             return_value=768,
@@ -226,3 +226,164 @@ class TestBootGating:
         assert 'os.environ.setdefault("AUTO_VECTOR_SCHEMA", "false")' in (
             conftest.read_text()
         )
+
+
+@pytest.mark.unit
+class TestBootDoesNotLoadTheModel:
+    """The hook needs an integer, not an inference session.
+
+    It used to build the embeddings instance to read ``.dimension`` off it,
+    loading several hundred MB of ONNX into every API and worker process at
+    import. For a model the registry describes that is a lookup.
+    """
+
+    def _run(self, registry_dim, loader):
+        conn = MagicMock()
+        conn.cursor.return_value = MagicMock()
+        with patch("psycopg.connect", return_value=conn), patch(
+            "application.vectorstore.model_registry.dimension_for",
+            return_value=registry_dim,
+        ), patch(
+            "application.vectorstore.base.build_local_embeddings", loader
+        ), patch(
+            "application.vectorstore.pgvector.PGVectorStore.create_schema"
+        ) as vector_schema, patch(
+            "application.vectorstore.pgvector.PGVectorStore.table_dimension",
+            return_value=None,
+        ):
+            ensure_vector_schema()
+        return vector_schema
+
+    def test_a_registered_model_is_never_constructed(self, vector_settings):
+        loader = MagicMock()
+        vector_schema = self._run(768, loader)
+        loader.assert_not_called()
+        assert vector_schema.call_args.kwargs["dimension"] == 768
+
+    def test_an_unregistered_model_still_falls_back_to_loading(self, vector_settings):
+        loader = MagicMock(return_value=_embeddings(1024))
+        vector_schema = self._run(None, loader)
+        loader.assert_called_once()
+        assert vector_schema.call_args.kwargs["dimension"] == 1024
+
+
+@pytest.mark.unit
+class TestUnknownWidthIsProbed:
+    """A remote server's width is only knowable by asking it.
+
+    ``RemoteEmbeddings`` reports ``None`` until its first call, so sizing the
+    table from the attribute alone fell back to 768 and skipped the mismatch
+    check — the silent ``vector(768)`` column this hook exists to prevent.
+    """
+
+    def _run(self, remote, table_dimension=1024):
+        conn = MagicMock()
+        conn.cursor.return_value = MagicMock()
+        with patch("psycopg.connect", return_value=conn), patch(
+            "application.vectorstore.model_registry.dimension_for", return_value=None
+        ), patch(
+            "application.vectorstore.base.build_local_embeddings", return_value=remote
+        ), patch(
+            "application.vectorstore.pgvector.PGVectorStore.create_schema"
+        ) as vector_schema, patch(
+            "application.vectorstore.pgvector.PGVectorStore.table_dimension",
+            return_value=table_dimension,
+        ):
+            try:
+                ensure_vector_schema()
+                raised = False
+            except RuntimeError:
+                raised = True
+        return vector_schema, raised
+
+    @staticmethod
+    def _remote(width=None, error=None):
+        remote = MagicMock()
+        remote.dimension = None
+        remote.embed_query.side_effect = error or (lambda _text: [0.0] * width)
+        return remote
+
+    def test_the_table_is_sized_from_the_probe(self, vector_settings):
+        remote = self._remote(width=1024)
+        vector_schema, _ = self._run(remote, table_dimension=1024)
+        remote.embed_query.assert_called_once()
+        assert vector_schema.call_args.kwargs["dimension"] == 1024
+
+    def test_the_probe_restores_the_mismatch_check(self, vector_settings):
+        _, raised = self._run(self._remote(width=768), table_dimension=1024)
+        assert raised, "a 768-dim model against a vector(1024) table must fail loudly"
+
+    def test_an_unreachable_server_does_not_block_boot(self, vector_settings):
+        vector_schema, raised = self._run(
+            self._remote(error=ConnectionError("server down")), table_dimension=1024
+        )
+        assert not raised
+        assert vector_schema.call_args.kwargs["dimension"] == 768
+
+    def test_a_model_that_knows_its_width_is_not_probed(self, vector_settings):
+        local = MagicMock()
+        local.dimension = 384
+        vector_schema, _ = self._run(local, table_dimension=384)
+        local.embed_query.assert_not_called()
+        assert vector_schema.call_args.kwargs["dimension"] == 384
+
+
+@pytest.mark.unit
+class TestBootLoadedModelIsReleased:
+    """The width probe must not leave a model resident in a delegating process.
+
+    Reading ``.dimension`` off an unregistered model means loading it, and
+    ``EmbeddingsSingleton`` caches what it builds. In an API that delegates
+    every embed to the worker that cached copy is never called again — it is
+    several hundred megabytes held for the life of the process, which is the
+    cost ``EMBEDDINGS_DELEGATE_TO_WORKER`` exists to avoid.
+    """
+
+    def _run(self, vector_settings, *, delegate, base_url=None):
+        from application.vectorstore.base import EmbeddingsSingleton
+
+        monkeyed = _embeddings(1024)
+        conn = MagicMock()
+        conn.cursor.return_value = MagicMock()
+        EmbeddingsSingleton._instances.pop("test-model", None)
+
+        def _build(*_args, **_kwargs):
+            EmbeddingsSingleton._instances["test-model"] = monkeyed
+            return monkeyed
+
+        with patch.object(
+            vector_settings, "EMBEDDINGS_DELEGATE_TO_WORKER", delegate
+        ), patch.object(
+            vector_settings, "EMBEDDINGS_BASE_URL", base_url
+        ), patch("psycopg.connect", return_value=conn), patch(
+            "application.vectorstore.model_registry.dimension_for", return_value=None
+        ), patch(
+            "application.vectorstore.base.build_local_embeddings", side_effect=_build
+        ), patch(
+            "application.vectorstore.pgvector.PGVectorStore.create_schema"
+        ) as vector_schema, patch(
+            "application.vectorstore.pgvector.PGVectorStore.table_dimension",
+            return_value=1024,
+        ):
+            ensure_vector_schema()
+        try:
+            return vector_schema, "test-model" in EmbeddingsSingleton._instances
+        finally:
+            EmbeddingsSingleton._instances.pop("test-model", None)
+
+    def test_a_delegating_process_does_not_retain_it(self, vector_settings):
+        vector_schema, retained = self._run(vector_settings, delegate=True)
+        assert not retained, "a delegating API must not hold the model it probed"
+        assert vector_schema.call_args.kwargs["dimension"] == 1024
+
+    def test_a_process_that_embeds_locally_keeps_it(self, vector_settings):
+        _, retained = self._run(vector_settings, delegate=False)
+        assert retained, "without delegation the model is used, so evicting it "\
+            "would only force a rebuild on the first query"
+
+    def test_a_remote_client_is_kept(self, vector_settings):
+        _, retained = self._run(
+            vector_settings, delegate=True, base_url="http://embeddings:8080"
+        )
+        assert retained, "a RemoteEmbeddings holds no model and is what the "\
+            "process goes on to use"

tests/storage/db/test_embeddings_pin.py

@@ -0,0 +1,131 @@
+"""Which embedding model an installation is pinned to."""
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from application.storage.db import embeddings_pin
+from application.storage.db.embeddings_pin import (
+    NOTICE_KEY,
+    PIN_KEY,
+    resolve_embeddings_pin,
+)
+from application.vectorstore.model_registry import DEFAULT_LEGACY, DEFAULT_NEW_INSTALL
+
+
+@pytest.fixture
+def store():
+    """An in-memory stand-in for the ``app_metadata`` key/value table."""
+    data = {}
+    repo = MagicMock()
+    repo.get.side_effect = data.get
+    repo.set.side_effect = lambda k, v: data.__setitem__(k, v)
+    repo.setdefault.side_effect = lambda k, v: data.setdefault(k, v)
+    repo._data = data
+    return repo
+
+
+def _run(store, *, has_sources, env_pinned=False, name="unset"):
+    session = MagicMock()
+    session.__enter__ = MagicMock(return_value=MagicMock())
+    session.__exit__ = MagicMock(return_value=False)
+    fields = {"EMBEDDINGS_NAME"} if env_pinned else set()
+    with patch.object(embeddings_pin.settings, "EMBEDDINGS_NAME", name), patch.object(
+        type(embeddings_pin.settings), "model_fields_set", property(lambda self: fields)
+    ), patch.object(embeddings_pin, "db_session", return_value=session), patch.object(
+        embeddings_pin, "AppMetadataRepository", return_value=store
+    ), patch.object(
+        embeddings_pin, "_has_sources", return_value=has_sources
+    ):
+        resolve_embeddings_pin()
+        return embeddings_pin.settings.EMBEDDINGS_NAME
+
+
+class TestFreshInstall:
+    def test_an_empty_installation_is_pinned_to_the_current_model(self, store):
+        assert _run(store, has_sources=False) == DEFAULT_NEW_INSTALL
+        assert store._data[PIN_KEY] == DEFAULT_NEW_INSTALL
+
+    def test_no_legacy_notice_is_printed(self, store, capsys):
+        _run(store, has_sources=False)
+        assert "reembed" not in capsys.readouterr().out
+        assert NOTICE_KEY not in store._data
+
+
+class TestExistingInstall:
+    """An index built by the old model must keep being read by it."""
+
+    def test_an_installation_with_sources_is_pinned_to_the_legacy_model(self, store):
+        assert _run(store, has_sources=True) == DEFAULT_LEGACY
+        assert store._data[PIN_KEY] == DEFAULT_LEGACY
+
+    def test_the_notice_names_the_migration_command(self, store, capsys):
+        _run(store, has_sources=True)
+        out = capsys.readouterr().out
+        assert "application.scripts.reembed" in out
+        assert DEFAULT_NEW_INSTALL in out
+
+    def test_the_notice_is_shown_only_once(self, store, capsys):
+        _run(store, has_sources=True)
+        capsys.readouterr()
+        store._data.pop(PIN_KEY)  # force the decision again
+        _run(store, has_sources=True)
+        assert "reembed" not in capsys.readouterr().out
+
+
+class TestPrecedence:
+    def test_a_stored_pin_survives_new_sources(self, store):
+        store._data[PIN_KEY] = DEFAULT_NEW_INSTALL
+        assert _run(store, has_sources=True) == DEFAULT_NEW_INSTALL
+
+    def test_the_environment_wins_and_nothing_is_stored(self, store):
+        assert _run(store, has_sources=True, env_pinned=True, name="my/model") == "my/model"
+        assert store._data == {}
+
+    def test_an_unreachable_database_leaves_the_default_alone(self):
+        with patch.object(embeddings_pin.settings, "EMBEDDINGS_NAME", "fallback"), patch.object(
+            type(embeddings_pin.settings), "model_fields_set", property(lambda self: set())
+        ), patch.object(embeddings_pin, "db_session", side_effect=OSError("no db")):
+            resolve_embeddings_pin()
+            assert embeddings_pin.settings.EMBEDDINGS_NAME == "fallback"
+
+
+class TestSourceModelMismatch:
+    """The only signal that an index is being queried by the wrong model."""
+
+    def _run(self, rows, active, log):
+        conn = MagicMock()
+        conn.execute.side_effect = [
+            MagicMock(scalar=MagicMock(return_value="sources")),
+            MagicMock(fetchall=MagicMock(return_value=rows)),
+        ]
+        session = MagicMock()
+        session.__enter__ = MagicMock(return_value=conn)
+        session.__exit__ = MagicMock(return_value=False)
+        with patch.object(embeddings_pin.settings, "EMBEDDINGS_NAME", active), patch.object(
+            embeddings_pin, "db_session", return_value=session
+        ):
+            embeddings_pin.warn_on_source_model_mismatch(log)
+
+    def test_a_different_model_is_reported_with_counts(self):
+        log = MagicMock()
+        self._run([(DEFAULT_LEGACY, 28)], DEFAULT_NEW_INSTALL, log)
+        message = log.warning.call_args.args[0] % log.warning.call_args.args[1:]
+        assert "28 built with" in message
+        assert "application.scripts.reembed" in message
+
+    def test_an_alias_is_not_a_mismatch(self):
+        """A stored alias and the canonical name are the same model."""
+        log = MagicMock()
+        self._run([("sentence-transformers/all-mpnet-base-v2", 28)], DEFAULT_LEGACY, log)
+        log.warning.assert_not_called()
+
+    def test_a_matching_model_says_nothing(self):
+        log = MagicMock()
+        self._run([(DEFAULT_NEW_INSTALL, 5)], DEFAULT_NEW_INSTALL, log)
+        log.warning.assert_not_called()
+
+    def test_two_unregistered_names_that_differ_are_a_mismatch(self):
+        log = MagicMock()
+        self._run([("some/other-model", 3)], "my/custom-model", log)
+        log.warning.assert_called_once()

tests/storage/db/test_migration_0031.py

@@ -0,0 +1,100 @@
+"""Migration round-trip test for 0031_token_usage_cache_tokens."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+from sqlalchemy import text
+
+
+pytestmark = pytest.mark.integration
+
+
+def _alembic_ini() -> Path:
+    return Path(__file__).resolve().parents[3] / "application" / "alembic.ini"
+
+
+def _run_alembic(url: str, *args: str) -> None:
+    subprocess.check_call(
+        [sys.executable, "-m", "alembic", "-c", str(_alembic_ini()), *args],
+        timeout=60,
+        env={**os.environ, "POSTGRES_URI": url},
+    )
+
+
+def _alembic_heads(url: str) -> list[str]:
+    out = subprocess.check_output(
+        [sys.executable, "-m", "alembic", "-c", str(_alembic_ini()), "heads"],
+        timeout=60,
+        env={**os.environ, "POSTGRES_URI": url},
+        text=True,
+    )
+    return [line for line in out.splitlines() if line.strip()]
+
+
+def _alembic_version(conn) -> str:
+    return conn.execute(text("SELECT version_num FROM alembic_version")).scalar()
+
+
+def _column_exists(conn, table: str, column: str) -> bool:
+    row = conn.execute(
+        text(
+            "SELECT 1 FROM information_schema.columns "
+            "WHERE table_name = :t AND column_name = :c AND table_schema = 'public'"
+        ),
+        {"t": table, "c": column},
+    ).fetchone()
+    return row is not None
+
+
+_0031 = "0031_token_usage_cache_tokens"
+_0030 = "0030_superseded_messages"
+
+
+class TestMigration0031RoundTrip:
+    def test_single_head(self, pg_engine):
+        url = pg_engine.url.render_as_string(hide_password=False)
+        assert len(_alembic_heads(url)) == 1
+
+    def test_head_has_cache_columns(self, pg_engine):
+        with pg_engine.connect() as conn:
+            assert _alembic_version(conn) >= _0031
+            assert _column_exists(conn, "token_usage", "cached_tokens")
+            assert _column_exists(conn, "token_usage", "cache_write_tokens")
+
+    def test_downgrade_drops_then_upgrade_restores(self, pg_engine):
+        url = pg_engine.url.render_as_string(hide_password=False)
+        _run_alembic(url, "downgrade", _0030)
+        with pg_engine.connect() as conn:
+            assert _alembic_version(conn) == _0030
+            assert not _column_exists(conn, "token_usage", "cached_tokens")
+            assert not _column_exists(conn, "token_usage", "cache_write_tokens")
+        _run_alembic(url, "upgrade", "head")
+        with pg_engine.connect() as conn:
+            assert _alembic_version(conn) >= _0031
+            assert _column_exists(conn, "token_usage", "cached_tokens")
+
+    def test_existing_rows_read_null_cache_bins(self, pg_engine):
+        """Rows written before 0031 must read NULL (unknown), not 0."""
+        url = pg_engine.url.render_as_string(hide_password=False)
+        _run_alembic(url, "downgrade", _0030)
+        with pg_engine.begin() as conn:
+            conn.execute(
+                text(
+                    "INSERT INTO token_usage (user_id, prompt_tokens, generated_tokens) "
+                    "VALUES ('u-mig31', 10, 1)"
+                )
+            )
+        _run_alembic(url, "upgrade", "head")
+        with pg_engine.connect() as conn:
+            row = conn.execute(
+                text(
+                    "SELECT cached_tokens, cache_write_tokens FROM token_usage "
+                    "WHERE user_id = 'u-mig31'"
+                )
+            ).fetchone()
+            assert tuple(row) == (None, None)

tests/storage/test_local_storage.py

@@ -16,6 +16,13 @@ def local_storage(temp_base_dir):
     return LocalStorage(base_dir=temp_base_dir)
 
 
+@pytest.fixture
+def real_storage(tmp_path):
+    """Storage over a real directory, for the write paths worth exercising."""
+    base = os.path.realpath(str(tmp_path))
+    return LocalStorage(base_dir=base), base
+
+
 @pytest.mark.unit
 class TestLocalStorageInitialization:
 
@@ -37,50 +44,87 @@ class TestLocalStorageInitialization:
         with pytest.raises(ValueError, match="Path traversal detected"):
             local_storage._get_full_path("/absolute/path/test.txt")
 
-    @patch("os.makedirs")
-    @patch("builtins.open", new_callable=mock_open)
-    @patch("shutil.copyfileobj")
-    def test_save_file_creates_directory_and_saves(
-        self, mock_copyfileobj, mock_file, mock_makedirs, local_storage
-    ):
-        file_data = io.BytesIO(b"test content")
-        path = "documents/test.txt"
+    def test_save_file_creates_directory_and_saves(self, real_storage):
+        storage, base = real_storage
+        result = storage.save_file(io.BytesIO(b"test content"), "documents/test.txt")
 
-        result = local_storage.save_file(file_data, path)
-
-        expected_dir = os.path.join(os.path.realpath("/tmp/test_storage"), "documents")
-        expected_file = os.path.join(os.path.realpath("/tmp/test_storage"), "documents/test.txt")
-
-        assert mock_makedirs.call_count == 1
-        assert os.path.normpath(mock_makedirs.call_args[0][0]) == os.path.normpath(
-            expected_dir
-        )
-        assert mock_makedirs.call_args[1] == {"exist_ok": True}
-
-        assert mock_file.call_count == 1
-        assert os.path.normpath(mock_file.call_args[0][0]) == os.path.normpath(
-            expected_file
-        )
-        assert mock_file.call_args[0][1] == "wb"
-
-        mock_copyfileobj.assert_called_once_with(file_data, mock_file())
+        written = os.path.join(base, "documents/test.txt")
+        assert os.path.isfile(written)
+        with open(written, "rb") as f:
+            assert f.read() == b"test content"
         assert result == {"storage_type": "local"}
 
-    @patch("os.makedirs")
-    def test_save_file_with_save_method(self, mock_makedirs, local_storage):
-        file_data = MagicMock()
-        file_data.save = MagicMock()
-        path = "documents/test.txt"
+    def test_save_file_with_save_method(self, real_storage):
+        """Werkzeug's ``FileStorage.save`` accepts the destination handle."""
+        storage, base = real_storage
 
-        result = local_storage.save_file(file_data, path)
+        class _Uploaded:
+            def save(self, dst):
+                dst.write(b"from save()")
 
-        expected_file = os.path.join(os.path.realpath("/tmp/test_storage"), "documents/test.txt")
-        assert file_data.save.call_count == 1
-        assert os.path.normpath(file_data.save.call_args[0][0]) == os.path.normpath(
-            expected_file
-        )
+        result = storage.save_file(_Uploaded(), "documents/test.txt")
+
+        with open(os.path.join(base, "documents/test.txt"), "rb") as f:
+            assert f.read() == b"from save()"
         assert result == {"storage_type": "local"}
 
+    def test_save_file_replaces_existing_content(self, real_storage):
+        storage, base = real_storage
+        storage.save_file(io.BytesIO(b"first"), "a/b.bin")
+        storage.save_file(io.BytesIO(b"second"), "a/b.bin")
+
+        with open(os.path.join(base, "a/b.bin"), "rb") as f:
+            assert f.read() == b"second"
+
+    def test_failed_write_leaves_the_previous_file_intact(self, real_storage):
+        """The whole point of writing through a temp file.
+
+        ``reembed`` rewrites an index in place; a half-written ``index.faiss``
+        loads at neither the old width nor the new one, so an interrupted write
+        must leave the previous bytes untouched rather than truncate them.
+        """
+        storage, base = real_storage
+        storage.save_file(io.BytesIO(b"the good index"), "indexes/s1/index.faiss")
+
+        class _DiesHalfway:
+            def __init__(self):
+                self._served = False
+
+            def read(self, size=-1):
+                if self._served:
+                    raise OSError("connection reset")
+                self._served = True
+                return b"corrupt"
+
+        with pytest.raises(OSError, match="connection reset"):
+            storage.save_file(_DiesHalfway(), "indexes/s1/index.faiss")
+
+        with open(os.path.join(base, "indexes/s1/index.faiss"), "rb") as f:
+            assert f.read() == b"the good index"
+
+    def test_failed_write_leaves_no_temp_file_behind(self, real_storage):
+        storage, base = real_storage
+
+        class _Explodes:
+            def read(self, size=-1):
+                raise OSError("boom")
+
+        with pytest.raises(OSError):
+            storage.save_file(_Explodes(), "indexes/s1/index.faiss")
+
+        assert os.listdir(os.path.join(base, "indexes/s1")) == []
+
+    def test_save_file_keeps_the_existing_permissions(self, real_storage):
+        """``mkstemp`` is 0600; the replacement must not silently narrow access."""
+        storage, base = real_storage
+        storage.save_file(io.BytesIO(b"one"), "a/b.bin")
+        target = os.path.join(base, "a/b.bin")
+        os.chmod(target, 0o640)
+
+        storage.save_file(io.BytesIO(b"two"), "a/b.bin")
+
+        assert os.stat(target).st_mode & 0o777 == 0o640
+
     def test_save_file_with_absolute_path_outside_base_raises(self, local_storage):
         file_data = io.BytesIO(b"test content")
         path = "/absolute/path/test.txt"

tests/test_celery.py

@@ -1,4 +1,4 @@
-from unittest.mock import patch
+from unittest.mock import MagicMock, patch
 
 import pytest
 from application.celery_init import make_celery
@@ -222,3 +222,53 @@ def test_unparseable_file_raises_the_non_retryable_type():
 
     with pytest.raises(DocumentParseError, match="No text could be extracted"):
         embed_and_store_documents([], "/tmp", "src", None)
+
+
+@pytest.mark.unit
+class TestReclaimIsSkippedForEmbeds:
+    """The post-task heap reclaim must not run on the query hot path.
+
+    ``_reclaim_memory_after_task`` exists for the large transient allocations
+    docling/torch parsing makes. Query embedding became a Celery task, and a
+    full generational collect on a worker holding the ONNX model measured ~86 ms
+    against ~8 ms for the embed itself -- a 9x slowdown of the round trip for a
+    task that allocates a few kilobytes.
+    """
+
+    @staticmethod
+    def _collects(task_name):
+        from application.celery_init import _reclaim_memory_after_task
+
+        task = MagicMock()
+        task.name = task_name
+        with patch("application.celery_init.gc.collect") as collect, patch(
+            "application.celery_init._trim_native_heap"
+        ):
+            _reclaim_memory_after_task(task=task, task_id="t", state="SUCCESS")
+        return collect.called
+
+    def test_the_embed_task_is_skipped(self):
+        assert not self._collects("application.vectorstore.embeddings_tasks.embed_texts")
+
+    def test_parsing_still_reclaims(self):
+        assert self._collects("application.api.user.tasks.parse_document")
+
+    def test_ingest_still_reclaims(self):
+        assert self._collects("application.api.user.tasks.ingest")
+
+    def test_an_unnamed_sender_still_reclaims(self):
+        """Unknown callers keep the old behaviour rather than silently skipping."""
+        from application.celery_init import _reclaim_memory_after_task
+
+        with patch("application.celery_init.gc.collect") as collect, patch(
+            "application.celery_init._trim_native_heap"
+        ):
+            _reclaim_memory_after_task(task_id="t", state="SUCCESS")
+        assert collect.called
+
+    def test_the_skip_list_names_the_real_task(self):
+        """A renamed task must not silently start paying the collect again."""
+        from application.celery_init import _NO_RECLAIM_TASKS
+        from application.vectorstore.embeddings_delegated import EMBED_TASK
+
+        assert EMBED_TASK in _NO_RECLAIM_TASKS

tests/test_upload_limits.py

@@ -1,5 +1,6 @@
 """Tests for bounded user-upload stream helpers."""
 
+import codecs
 import io
 
 import pytest
@@ -37,3 +38,222 @@ def test_limited_text_read_decodes_incrementally_and_rejects_overflow():
 
     with pytest.raises(UploadTooLargeError):
         read_text_upload_limited(_ShortReadStream(b"12345"), max_bytes=4)
+
+
+# --- attachment type gate -------------------------------------------------
+#
+# ``SimpleDirectoryReader`` falls through to a plain-text ``open()`` for any
+# suffix without a parser. That is what a .py or a .log attachment relies on,
+# and it is also how a phone-uploaded video used to be "parsed" into
+# megabytes of binary garbage, truncated, and stored with
+# ``extraction.status == "ok"``. So a suffix with no parser is admitted on
+# content: text in, binary out.
+
+MP4_HEADER = b"\x00\x00\x00\x18ftypisom\x00\x00\x02\x00isomiso2avc1mp41"
+
+
+@pytest.mark.parametrize(
+    ("filename", "content"),
+    [
+        ("clip.mp4", MP4_HEADER),
+        ("movie.MOV", MP4_HEADER),
+        ("archive.zip", b"PK\x03\x04\x14\x00\x00\x00\x08\x00" + bytes(range(32))),
+        ("binary", bytes(range(32)) * 8),
+        ("trailing.", b"\x00\x01\x02\x03"),
+        ("x.tar.gz", b"\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03"),
+    ],
+)
+def test_enforce_parseable_attachment_rejects_binary_without_a_parser(
+    filename, content, tmp_path
+):
+    from application.upload_limits import (
+        enforce_parseable_attachment,
+        UnsupportedUploadTypeError,
+        unsupported_upload_message,
+    )
+
+    path = tmp_path / "staged.bin"
+    path.write_bytes(content)
+
+    with pytest.raises(UnsupportedUploadTypeError) as excinfo:
+        enforce_parseable_attachment(path, filename)
+    assert str(excinfo.value) == unsupported_upload_message(filename)
+    assert str(excinfo.value).startswith("Unsupported file type")
+
+
+def test_enforce_parseable_attachment_rejects_binary_named_as_text(tmp_path):
+    """.txt has no parser — it *is* the fallthrough — so it is sniffed like any suffix.
+
+    Renaming a video to notes.txt would otherwise walk straight back into the
+    bug this gate exists for.
+    """
+    from application.upload_limits import (
+        enforce_parseable_attachment,
+        UnsupportedUploadTypeError,
+    )
+
+    path = tmp_path / "notes.txt"
+    path.write_bytes(MP4_HEADER + bytes(range(256)) * 4)
+
+    with pytest.raises(UnsupportedUploadTypeError) as excinfo:
+        enforce_parseable_attachment(path, "notes.txt")
+    assert str(excinfo.value) == "Unsupported file type: .txt"
+
+
+@pytest.mark.parametrize(
+    "content",
+    [
+        codecs.BOM_UTF8 + "hello — Unicode\n".encode(),
+        codecs.BOM_UTF16_LE + "hello\n".encode("utf-16-le"),
+        codecs.BOM_UTF16_BE + "hello\n".encode("utf-16-be"),
+        codecs.BOM_UTF32_BE + "hello\n".encode("utf-32-be"),
+    ],
+)
+def test_enforce_parseable_attachment_accepts_bom_marked_unicode_text(
+    content, tmp_path
+):
+    """A UTF-16 .txt is half NUL bytes and still ordinary text — the BOM says so."""
+    from application.upload_limits import enforce_parseable_attachment
+
+    path = tmp_path / "notes.txt"
+    path.write_bytes(content)
+
+    enforce_parseable_attachment(path, "notes.txt")
+
+
+@pytest.mark.parametrize(
+    "bom",
+    [codecs.BOM_UTF8, codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE, codecs.BOM_UTF32_BE],
+)
+def test_enforce_parseable_attachment_rejects_binary_behind_a_bom(bom, tmp_path):
+    """A BOM says which encoding to read, not that the content is text.
+
+    Otherwise three prepended bytes buy any binary a pass.
+    """
+    from application.upload_limits import (
+        enforce_parseable_attachment,
+        UnsupportedUploadTypeError,
+    )
+
+    path = tmp_path / "notes.txt"
+    path.write_bytes(bom + MP4_HEADER + bytes(range(256)) * 8)
+
+    with pytest.raises(UnsupportedUploadTypeError):
+        enforce_parseable_attachment(path, "notes.txt")
+
+
+def test_enforce_parseable_attachment_uses_the_extractor_it_is_given(tmp_path):
+    """The worker holds the live parser table; a trimmed install must not admit on trust.
+
+    Without docling the fallback extractor has no .webp handler, so a .webp
+    would otherwise skip the content check and be read as plain text.
+    """
+    from application.upload_limits import (
+        enforce_parseable_attachment,
+        UnsupportedUploadTypeError,
+    )
+
+    path = tmp_path / "scan.webp"
+    path.write_bytes(b"RIFF\x00\x00\x00\x00WEBPVP8 " + bytes(range(256)))
+
+    # Default list: .webp is parser-backed, admitted on its name.
+    enforce_parseable_attachment(path, "scan.webp")
+
+    # The extractor actually loaded has no .webp parser.
+    with pytest.raises(UnsupportedUploadTypeError):
+        enforce_parseable_attachment(path, "scan.webp", {".pdf", ".docx"})
+
+
+@pytest.mark.parametrize(
+    "filename",
+    [
+        "Report.PDF",
+        "photo.JPG",
+        "slides.pptx",
+        "voice.ogg",
+        "page.xhtml",
+        "doc.adoc",
+        "scan.webp",
+        "fax.tiff",
+        "subs.vtt",
+        "feed.xml",
+    ],
+)
+def test_enforce_parseable_attachment_accepts_parser_backed_types(filename, tmp_path):
+    """A parser-backed suffix is admitted on its name — a PDF is binary and parses fine."""
+    from application.upload_limits import enforce_parseable_attachment
+
+    path = tmp_path / "staged.bin"
+    path.write_bytes(MP4_HEADER)
+
+    enforce_parseable_attachment(path, filename)
+
+
+@pytest.mark.parametrize(
+    "filename",
+    [
+        "notes.txt",
+        "main.py",
+        "server.log",
+        "config.yaml",
+        "query.sql",
+        "Dockerfile",
+        "notes.unknown",
+    ],
+)
+def test_enforce_parseable_attachment_accepts_text_without_a_parser(filename, tmp_path):
+    """The plain-text fallthrough reads these correctly, so they must stay allowed."""
+    from application.upload_limits import enforce_parseable_attachment
+
+    path = tmp_path / "staged.txt"
+    path.write_text("def main():\n\treturn 'café — ok'\n", encoding="utf-8")
+
+    enforce_parseable_attachment(path, filename)
+
+
+@pytest.mark.parametrize(
+    ("sample", "expected"),
+    [
+        (b"", True),
+        (b"plain text\n", True),
+        ("café — em dash\n".encode(), True),
+        (b"\x1b[31mred log line\x1b[0m\n", True),
+        (codecs.BOM_UTF16_LE + "hi\n".encode("utf-16-le"), True),
+        (codecs.BOM_UTF8 + b"hi\n", True),
+        (codecs.BOM_UTF32_LE + "hi\n".encode("utf-32-le"), True),
+        # A BOM in front of binary is still binary.
+        (codecs.BOM_UTF8 + b"\x00\x01\x02", False),
+        (codecs.BOM_UTF16_LE + MP4_HEADER, False),
+        (b"text\x00with nul", False),
+        (bytes(range(32)) * 4, False),
+        (b"\x7f\x7f\x7f\x7f" + b"a" * 16, False),
+    ],
+)
+def test_looks_like_text(sample, expected):
+    from application.upload_limits import looks_like_text
+
+    assert looks_like_text(sample) is expected
+
+
+def test_file_looks_like_text_only_samples_the_head(tmp_path):
+    """Binary past the sampled head is the parser's problem, not the gate's."""
+    from application.upload_limits import file_looks_like_text
+
+    path = tmp_path / "staged.log"
+    path.write_bytes(b"a" * 9000 + b"\x00" * 100)
+
+    assert file_looks_like_text(path) is True
+
+
+def test_file_looks_like_text_allows_an_unreadable_file(tmp_path):
+    from application.upload_limits import file_looks_like_text
+
+    assert file_looks_like_text(tmp_path / "missing.txt") is True
+
+
+def test_unsupported_upload_message_names_the_extension():
+    from application.upload_limits import unsupported_upload_message
+
+    assert unsupported_upload_message("clip.mp4") == "Unsupported file type: .mp4"
+    assert unsupported_upload_message("Clip.MP4") == "Unsupported file type: .mp4"
+    assert unsupported_upload_message("binary") == "Unsupported file type: (no extension)"

tests/test_usage.py

@@ -600,3 +600,115 @@ class TestCountPromptTokens:
         ]
         tokens = _count_prompt_tokens(messages, tools=None)
         assert tokens > 0
+
+
+# ── prompt-cache breakdown ────────────────────────────────────────────────────
+#
+# Providers report ``cached_tokens`` (and, on newer OpenAI-family models,
+# ``cache_write_tokens``) as breakdowns of ``prompt_tokens``. The prompt bin
+# stays the provider total (never subtract the details back out); the two
+# sub-bins ride alongside so persistence and the finish events can chart a
+# cache hit rate.
+
+
+class _ReportingLLM:
+    decoded_token = {"sub": "user_1"}
+    user_api_key = None
+    agent_id = None
+    token_usage = {"prompt_tokens": 0, "generated_tokens": 0}
+
+    def __init__(self, details):
+        self._last_usage = {
+            "prompt_tokens": 1000,
+            "completion_tokens": 10,
+            "total_tokens": 1010,
+        }
+        if details is not None:
+            self._last_usage["prompt_tokens_details"] = details
+        self._last_usage_claimed = False
+        self.emitted = []
+
+    def _emit_gen_finished_log(self, model, **kwargs):
+        self.emitted.append(kwargs)
+
+
+@pytest.mark.unit
+def test_prefer_provider_usage_carries_cache_bins():
+    from application.usage import _prefer_provider_usage
+
+    llm = _ReportingLLM({"cached_tokens": 800, "cache_write_tokens": 100})
+    usage = _prefer_provider_usage(llm, {"prompt_tokens": 1, "generated_tokens": 1})
+    assert usage["prompt_tokens"] == 1000
+    assert usage["generated_tokens"] == 10
+    assert usage["cached_tokens"] == 800
+    assert usage["cache_write_tokens"] == 100
+
+
+@pytest.mark.unit
+def test_prefer_provider_usage_maps_anthropic_cache_creation_to_writes():
+    from application.usage import _prefer_provider_usage
+
+    llm = _ReportingLLM({"cached_tokens": 3, "cache_creation_tokens": 9})
+    usage = _prefer_provider_usage(llm, {"prompt_tokens": 1, "generated_tokens": 1})
+    assert usage["cached_tokens"] == 3
+    assert usage["cache_write_tokens"] == 9
+
+
+@pytest.mark.unit
+def test_prefer_provider_usage_without_details_has_no_cache_keys():
+    from application.usage import _prefer_provider_usage
+
+    llm = _ReportingLLM(None)
+    usage = _prefer_provider_usage(llm, {"prompt_tokens": 1, "generated_tokens": 1})
+    assert "cached_tokens" not in usage
+    assert "cache_write_tokens" not in usage
+
+
+@pytest.mark.unit
+def test_prefer_provider_usage_keeps_the_rest_of_the_call_record():
+    from application.usage import _prefer_provider_usage
+
+    llm = _ReportingLLM(None)
+    usage = _prefer_provider_usage(
+        llm, {"prompt_tokens": 1, "generated_tokens": 1, "model": "m"}
+    )
+    assert usage["model"] == "m"
+
+
+@pytest.mark.unit
+def test_decorator_persists_and_emits_cache_bins(monkeypatch):
+    _install_fake_token_repo(monkeypatch)
+    llm = _ReportingLLM({"cached_tokens": 800, "cache_write_tokens": 100})
+
+    @gen_token_usage
+    def wrapped(self, model, messages, stream, tools, **kwargs):
+        _ = (model, messages, stream, tools, kwargs)
+        return "ok"
+
+    wrapped(llm, "m", [{"role": "user", "content": "hi"}], False, None)
+
+    row = _FakeTokenUsageRepo.last_instance.inserted[0]
+    assert row["prompt_tokens"] == 1000
+    assert row["cached_tokens"] == 800
+    assert row["cache_write_tokens"] == 100
+    assert llm.emitted[0]["cached_tokens"] == 800
+    assert llm.emitted[0]["cache_write_tokens"] == 100
+
+
+@pytest.mark.unit
+def test_decorator_omits_cache_bins_when_provider_reports_none(monkeypatch):
+    _install_fake_token_repo(monkeypatch)
+    llm = _ReportingLLM(None)
+
+    @gen_token_usage
+    def wrapped(self, model, messages, stream, tools, **kwargs):
+        _ = (model, messages, stream, tools, kwargs)
+        return "ok"
+
+    wrapped(llm, "m", [{"role": "user", "content": "hi"}], False, None)
+
+    row = _FakeTokenUsageRepo.last_instance.inserted[0]
+    assert row["cached_tokens"] is None
+    assert row["cache_write_tokens"] is None
+    assert llm.emitted[0]["cached_tokens"] is None
+    assert llm.emitted[0]["cache_write_tokens"] is None

tests/vectorstore/test_base.py

@@ -263,11 +263,11 @@ class TestEmbeddingsSingleton:
     def test_get_instance_hf_ignores_positional_key(
         self, mock_get_wrapper, mock_settings
     ):
-        """A stray key must not reach the zero-arg HuggingFace factory.
+        """A stray key must not reach the wrapper for a registered model.
 
-        The factories are ``lambda: EmbeddingsWrapper(...)``, so a caller that
-        passed ``settings.EMBEDDINGS_KEY`` positionally used to blow up with
-        ``TypeError: <lambda>() takes 0 positional arguments``.
+        Registered models take their whole configuration from the registry, so
+        a caller that passes ``settings.EMBEDDINGS_KEY`` positionally (as the
+        vector stores do) must have it dropped rather than forwarded.
         """
         mock_settings.EMBEDDINGS_BASE_URL = None
         mock_wrapper_cls = Mock()
@@ -278,9 +278,8 @@ class TestEmbeddingsSingleton:
         result = EmbeddingsSingleton.get_instance(HF_MPNET, None)
 
         assert result is mock_instance
-        mock_wrapper_cls.assert_called_once_with(
-            "sentence-transformers/all-mpnet-base-v2"
-        )
+        # The configured name is passed through; the registry maps it to a repo.
+        mock_wrapper_cls.assert_called_once_with(HF_MPNET)
 
     @patch("application.vectorstore.base.settings")
     @patch("application.vectorstore.base._get_embeddings_wrapper")
@@ -293,9 +292,7 @@ class TestEmbeddingsSingleton:
 
         EmbeddingsSingleton.get_instance(HF_MPNET, openai_api_key="sk-nope")
 
-        mock_wrapper_cls.assert_called_once_with(
-            "sentence-transformers/all-mpnet-base-v2"
-        )
+        mock_wrapper_cls.assert_called_once_with(HF_MPNET)
 
 
 # --- BaseVectorStore ---
@@ -401,12 +398,15 @@ class TestBaseVectorStore:
 
     @patch("application.vectorstore.base.settings")
     @patch("application.vectorstore.base.EmbeddingsSingleton.get_instance")
-    @patch("os.path.exists")
-    def test_get_embeddings_huggingface_local_model(
-        self, mock_exists, mock_get_instance, mock_settings
+    def test_get_embeddings_registered_model_passes_configured_name(
+        self, mock_get_instance, mock_settings
     ):
+        """No bundled-path branch any more: the name goes straight through.
+
+        FastEmbed resolves artifacts through its own cache (warmed in the
+        image), so the old ``/app/models/...`` probe has no job to do.
+        """
         mock_settings.EMBEDDINGS_BASE_URL = None
-        mock_exists.side_effect = lambda p: p == "/app/models/all-mpnet-base-v2"
         mock_emb = Mock()
         mock_get_instance.return_value = mock_emb
 
@@ -415,7 +415,9 @@ class TestBaseVectorStore:
             "huggingface_sentence-transformers/all-mpnet-base-v2"
         )
         assert result is mock_emb
-        mock_get_instance.assert_called_with("/app/models/all-mpnet-base-v2")
+        mock_get_instance.assert_called_with(
+            "huggingface_sentence-transformers/all-mpnet-base-v2"
+        )
 
     @patch("application.vectorstore.base.settings")
     @patch("application.vectorstore.base.EmbeddingsSingleton.get_instance")
@@ -518,18 +520,18 @@ class TestGetEmbeddingsResolver:
 
     @patch("application.vectorstore.base.settings")
     @patch("application.vectorstore.base._get_embeddings_wrapper")
-    @patch("os.path.exists")
-    def test_uses_local_model_path_and_caches_it(
-        self, mock_exists, mock_get_wrapper, mock_settings
+    def test_repeated_resolution_loads_one_model(
+        self, mock_get_wrapper, mock_settings
     ):
-        """With the bundled model present, the instance is keyed by its path.
+        """A second call must not load a second copy of the model.
 
-        A second call must not load a second copy of the model.
+        The instance is keyed by the configured name. It used to be keyed by a
+        bundled filesystem path when one happened to exist, which meant the
+        same model could be cached twice under two keys.
         """
         mock_settings.EMBEDDINGS_BASE_URL = None
         mock_settings.EMBEDDINGS_NAME = HF_MPNET
         mock_settings.EMBEDDINGS_KEY = None
-        mock_exists.side_effect = lambda path: path == LOCAL_MPNET
         mock_wrapper_cls = Mock()
         mock_wrapper_cls.return_value = Mock()
         mock_get_wrapper.return_value = mock_wrapper_cls
@@ -538,8 +540,8 @@ class TestGetEmbeddingsResolver:
         second = get_embeddings()
 
         assert first is second
-        assert set(EmbeddingsSingleton._instances) == {LOCAL_MPNET}
-        mock_wrapper_cls.assert_called_once_with(LOCAL_MPNET)
+        assert set(EmbeddingsSingleton._instances) == {HF_MPNET}
+        mock_wrapper_cls.assert_called_once_with(HF_MPNET)
 
     @patch("application.vectorstore.base.settings")
     def test_remote_when_base_url_configured(self, mock_settings):
@@ -589,6 +591,49 @@ class TestGetEmbeddingsResolver:
             "openai_text-embedding-ada-002", model="embed-deploy"
         )
 
+    @patch("application.vectorstore.base.settings")
+    @patch("application.vectorstore.base.EmbeddingsSingleton.get_instance")
+    def test_openai_alias_also_reaches_the_azure_deployment(
+        self, mock_get_instance, mock_settings
+    ):
+        """The registry accepts the bare alias, so the key handling must too.
+
+        Matching on the canonical string alone sent the alias down the generic
+        branch, where the deployment name is never passed and Azure answers
+        every embed with DeploymentNotFound.
+        """
+        mock_settings.EMBEDDINGS_BASE_URL = None
+        mock_settings.OPENAI_API_BASE = "https://azure.openai.com"
+        mock_settings.OPENAI_API_VERSION = "2023-05-15"
+        mock_settings.AZURE_DEPLOYMENT_NAME = "deploy"
+        mock_settings.AZURE_EMBEDDINGS_DEPLOYMENT_NAME = "embed-deploy"
+        mock_settings.EMBEDDINGS_NAME = "text-embedding-ada-002"
+        mock_settings.EMBEDDINGS_KEY = "sk-key"
+
+        get_embeddings()
+
+        mock_get_instance.assert_called_once_with(
+            "text-embedding-ada-002", model="embed-deploy"
+        )
+
+    @patch("application.vectorstore.base.settings")
+    @patch("application.vectorstore.base.EmbeddingsSingleton.get_instance")
+    def test_openai_name_is_matched_case_insensitively(
+        self, mock_get_instance, mock_settings
+    ):
+        mock_settings.EMBEDDINGS_BASE_URL = None
+        mock_settings.OPENAI_API_BASE = None
+        mock_settings.OPENAI_API_VERSION = None
+        mock_settings.AZURE_DEPLOYMENT_NAME = None
+        mock_settings.EMBEDDINGS_NAME = "OpenAI_Text-Embedding-Ada-002"
+        mock_settings.EMBEDDINGS_KEY = "sk-from-settings"
+
+        get_embeddings()
+
+        mock_get_instance.assert_called_once_with(
+            "OpenAI_Text-Embedding-Ada-002", openai_api_key="sk-from-settings"
+        )
+
     @patch("application.vectorstore.base.settings")
     @patch("application.vectorstore.base.EmbeddingsSingleton.get_instance")
     def test_explicit_arguments_win_over_settings(

tests/vectorstore/test_embeddings_delegated.py

@@ -0,0 +1,380 @@
+"""Query embedding runs on the worker so the API holds no model."""
+
+import threading
+import time
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from application.vectorstore import base
+from application.vectorstore.embeddings_delegated import EMBED_TASK, DelegatedEmbeddings
+
+
+@pytest.fixture(autouse=True)
+def _clear_singleton():
+    base.EmbeddingsSingleton._instances.clear()
+    yield
+    base.EmbeddingsSingleton._instances.clear()
+
+
+@pytest.fixture
+def not_in_worker():
+    with patch("application.vectorstore.embeddings_delegated._in_worker", return_value=False):
+        yield
+
+
+class TestDispatch:
+    def test_query_is_embedded_on_the_worker(self, not_in_worker):
+        celery = MagicMock()
+        celery.send_task.return_value.get.return_value = [[0.1, 0.2, 0.3]]
+        with patch("application.celery_init.celery", celery):
+            vector = DelegatedEmbeddings("some/model").embed_query("hello")
+        assert vector == [0.1, 0.2, 0.3]
+        assert celery.send_task.call_args.args[0] == EMBED_TASK
+        assert celery.send_task.call_args.kwargs["args"] == [["hello"], "some/model"]
+
+    def test_routed_to_the_embeddings_queue(self, not_in_worker):
+        celery = MagicMock()
+        celery.send_task.return_value.get.return_value = [[0.0]]
+        with patch("application.celery_init.celery", celery):
+            with patch.object(base.settings, "EMBEDDINGS_QUEUE", "embeddings"):
+                DelegatedEmbeddings("some/model").embed_query("hi")
+        assert celery.send_task.call_args.kwargs["queue"] == "embeddings"
+
+    def test_no_worker_gives_an_actionable_error(self, not_in_worker):
+        celery = MagicMock()
+        celery.send_task.return_value.get.side_effect = TimeoutError("no worker")
+        with patch("application.celery_init.celery", celery):
+            with pytest.raises(RuntimeError) as excinfo:
+                DelegatedEmbeddings("some/model").embed_query("hi")
+        message = str(excinfo.value)
+        assert "EMBEDDINGS_DELEGATE_TO_WORKER=false" in message
+        assert "EMBEDDINGS_BASE_URL" in message
+
+    def test_empty_input_never_reaches_the_broker(self, not_in_worker):
+        celery = MagicMock()
+        with patch("application.celery_init.celery", celery):
+            assert DelegatedEmbeddings("some/model").embed_documents([]) == []
+        celery.send_task.assert_not_called()
+
+
+class TestInsideAWorker:
+    """Dispatching from inside a task would queue work behind itself."""
+
+    def test_a_running_task_embeds_locally(self):
+        local = MagicMock()
+        local.embed_documents.return_value = [[1.0, 2.0]]
+        celery = MagicMock()
+        with patch("application.vectorstore.embeddings_delegated._in_worker", return_value=True):
+            with patch("application.vectorstore.base.build_local_embeddings", return_value=local):
+                with patch("application.celery_init.celery", celery):
+                    vector = DelegatedEmbeddings("some/model").embed_query("hi")
+        assert vector == [1.0, 2.0]
+        celery.send_task.assert_not_called()
+
+    def test_the_local_model_is_built_once(self):
+        local = MagicMock()
+        local.embed_documents.return_value = [[1.0]]
+        builder = MagicMock(return_value=local)
+        client = DelegatedEmbeddings("some/model")
+        with patch("application.vectorstore.embeddings_delegated._in_worker", return_value=True):
+            with patch("application.vectorstore.base.build_local_embeddings", builder):
+                client.embed_query("a")
+                client.embed_query("b")
+        builder.assert_called_once()
+
+
+class TestDimension:
+    def test_registry_width_costs_no_round_trip(self):
+        celery = MagicMock()
+        with patch("application.celery_init.celery", celery):
+            client = DelegatedEmbeddings("ibm-granite/granite-embedding-311m-multilingual-r2")
+            assert client.dimension == 768
+        celery.send_task.assert_not_called()
+
+    def test_unknown_width_is_probed_once(self, not_in_worker):
+        celery = MagicMock()
+        celery.send_task.return_value.get.return_value = [[0.0] * 1024]
+        with patch("application.celery_init.celery", celery):
+            client = DelegatedEmbeddings("some/unregistered")
+            assert client.dimension == 1024
+            assert client.dimension == 1024
+        celery.send_task.assert_called_once()
+
+    def test_an_unreachable_worker_reports_no_width(self, not_in_worker):
+        celery = MagicMock()
+        celery.send_task.return_value.get.side_effect = TimeoutError("down")
+        with patch("application.celery_init.celery", celery):
+            assert DelegatedEmbeddings("some/unregistered").dimension is None
+
+
+class TestGetEmbeddingsDispatch:
+    def test_delegates_when_enabled(self):
+        with patch.object(base.settings, "EMBEDDINGS_BASE_URL", None):
+            with patch.object(base.settings, "EMBEDDINGS_DELEGATE_TO_WORKER", True):
+                assert isinstance(base.get_embeddings("some/model"), DelegatedEmbeddings)
+
+    def test_remote_url_wins_over_delegation(self):
+        with patch.object(base.settings, "EMBEDDINGS_BASE_URL", "http://embed.local"):
+            with patch.object(base.settings, "EMBEDDINGS_DELEGATE_TO_WORKER", True):
+                assert isinstance(base.get_embeddings("some/model"), base.RemoteEmbeddings)
+
+    def test_disabled_loads_in_process(self):
+        with patch.object(base.settings, "EMBEDDINGS_BASE_URL", None):
+            with patch.object(base.settings, "EMBEDDINGS_DELEGATE_TO_WORKER", False):
+                with patch.object(base.EmbeddingsSingleton, "get_instance") as get_instance:
+                    base.get_embeddings("some/model")
+        get_instance.assert_called_once()
+
+    def test_the_delegating_client_is_shared(self):
+        with patch.object(base.settings, "EMBEDDINGS_BASE_URL", None):
+            with patch.object(base.settings, "EMBEDDINGS_DELEGATE_TO_WORKER", True):
+                assert base.get_embeddings("some/model") is base.get_embeddings("some/model")
+
+
+class TestFailureCooldown:
+    """One dead-worker timeout per retrieval, not one per source.
+
+    ``fanout.embed_questions`` swallows a dispatch failure and lets every store
+    embed its own query, so without a latch a single chat request pays
+    ``EMBEDDINGS_DELEGATE_TIMEOUT`` once in the fan-out and again per source.
+    A missing worker is a property of the deployment, not of the call.
+    """
+
+    @staticmethod
+    def _celery(side_effect):
+        result = MagicMock()
+        result.get.side_effect = side_effect
+        celery = MagicMock()
+        celery.send_task.return_value = result
+        return celery, result
+
+    def test_only_the_first_call_waits_out_the_timeout(self, not_in_worker):
+        celery, _ = self._celery(TimeoutError("no worker"))
+        embeddings = DelegatedEmbeddings("granite-311m")
+        with patch.dict("sys.modules", {"application.celery_init": MagicMock(celery=celery)}):
+            for _ in range(4):
+                with pytest.raises(RuntimeError):
+                    embeddings.embed_query("q")
+        assert celery.send_task.call_count == 1
+
+    def test_the_fast_failure_still_names_the_remedy(self, not_in_worker):
+        celery, _ = self._celery(TimeoutError("no worker"))
+        embeddings = DelegatedEmbeddings("granite-311m")
+        with patch.dict("sys.modules", {"application.celery_init": MagicMock(celery=celery)}):
+            with pytest.raises(RuntimeError):
+                embeddings.embed_query("q")
+            with pytest.raises(RuntimeError, match="EMBEDDINGS_DELEGATE_TO_WORKER=false"):
+                embeddings.embed_query("q")
+
+    def test_the_latch_clears_once_the_worker_answers(self, not_in_worker):
+        celery, result = self._celery(TimeoutError("no worker"))
+        embeddings = DelegatedEmbeddings("granite-311m")
+        with patch.dict("sys.modules", {"application.celery_init": MagicMock(celery=celery)}):
+            with pytest.raises(RuntimeError):
+                embeddings.embed_query("q")
+            embeddings._failed_at = None  # stand in for the cooldown elapsing
+            result.get.side_effect = None
+            result.get.return_value = [[0.5, 0.5]]
+            assert embeddings.embed_query("q") == [0.5, 0.5]
+        assert embeddings._cooldown_remaining() == 0.0
+
+    def test_a_healthy_worker_is_never_latched(self, not_in_worker):
+        celery, result = self._celery(None)
+        result.get.return_value = [[0.1, 0.2]]
+        embeddings = DelegatedEmbeddings("granite-311m")
+        with patch.dict("sys.modules", {"application.celery_init": MagicMock(celery=celery)}):
+            for _ in range(3):
+                assert embeddings.embed_query("q") == [0.1, 0.2]
+        assert celery.send_task.call_count == 3
+
+
+class TestTheConcurrentFirstWave:
+    """The latch cannot cover requests already in flight beside the first one.
+
+    Nothing is latched until that first ``get()`` returns, so every thread in
+    the opening wave would otherwise block for the full
+    ``EMBEDDINGS_DELEGATE_TIMEOUT`` at once -- at the shipped 60s across a 96
+    thread WSGI pool, an API that serves nothing at all.
+    """
+
+    @staticmethod
+    def _blocking_celery(release, outcome):
+        """A worker whose ``get`` blocks until ``release`` is set."""
+        def get(timeout=None):
+            release.wait(5)
+            if isinstance(outcome, Exception):
+                raise outcome
+            return outcome
+
+        result = MagicMock()
+        result.get.side_effect = get
+        celery = MagicMock()
+        celery.send_task.return_value = result
+        return celery
+
+    def _race(self, celery, embeddings, release, threads=8):
+        """Start ``threads`` embeds, let them pile up, then unblock the prober."""
+        errors, values = [], []
+        started = threading.Barrier(threads + 1)
+
+        def call():
+            started.wait(5)
+            try:
+                values.append(embeddings.embed_query("q"))
+            except Exception as exc:  # noqa: BLE001 -- recorded for the assertions
+                errors.append(exc)
+
+        with patch.dict("sys.modules", {"application.celery_init": MagicMock(celery=celery)}):
+            workers = [threading.Thread(target=call) for _ in range(threads)]
+            for worker in workers:
+                worker.start()
+            started.wait(5)
+            time.sleep(0.1)  # let the followers reach the probe gate
+            release.set()
+            for worker in workers:
+                worker.join(10)
+        return values, errors
+
+    def test_only_one_caller_waits_on_an_unproven_worker(self, not_in_worker):
+        release = threading.Event()
+        celery = self._blocking_celery(release, TimeoutError("no worker"))
+        embeddings = DelegatedEmbeddings("granite-311m")
+        with patch(
+            "application.vectorstore.embeddings_delegated._PROBE_WAIT", 0.05
+        ):
+            values, errors = self._race(celery, embeddings, release)
+
+        assert values == []
+        assert len(errors) == 8
+        # One probe published; the rest gave up without their own round trip.
+        assert celery.send_task.call_count == 1
+        assert sum("still unanswered" in str(e) for e in errors) == 7
+
+    def test_the_fast_failure_still_names_the_remedy(self, not_in_worker):
+        release = threading.Event()
+        celery = self._blocking_celery(release, TimeoutError("no worker"))
+        embeddings = DelegatedEmbeddings("granite-311m")
+        with patch("application.vectorstore.embeddings_delegated._PROBE_WAIT", 0.05):
+            _, errors = self._race(celery, embeddings, release, threads=3)
+        assert all("EMBEDDINGS_DELEGATE_TO_WORKER=false" in str(e) for e in errors)
+
+    def test_a_healthy_worker_serves_the_whole_wave(self, not_in_worker):
+        release = threading.Event()
+        celery = self._blocking_celery(release, [[0.1, 0.2]])
+        embeddings = DelegatedEmbeddings("granite-311m")
+        values, errors = self._race(celery, embeddings, release)
+
+        assert errors == []
+        assert values == [[0.1, 0.2]] * 8
+        # The probe proves the worker, then every follower dispatches for real.
+        assert celery.send_task.call_count == 8
+        assert embeddings._verified is True
+
+    def test_a_proven_worker_adds_no_gate(self, not_in_worker):
+        """After one success the probe is out of the path entirely."""
+        release = threading.Event()
+        release.set()
+        celery = self._blocking_celery(release, [[0.3]])
+        embeddings = DelegatedEmbeddings("granite-311m")
+        with patch.dict("sys.modules", {"application.celery_init": MagicMock(celery=celery)}):
+            embeddings.embed_query("warm")
+        assert embeddings._verified is True
+
+        with patch.object(embeddings, "_state_lock") as lock:
+            with patch.dict(
+                "sys.modules", {"application.celery_init": MagicMock(celery=celery)}
+            ):
+                embeddings.embed_query("q")
+            lock.__enter__.assert_not_called()
+
+    def test_a_proven_worker_that_dies_is_gated_again(self, not_in_worker):
+        """The proof must not outlive the worker that supplied it.
+
+        A worker that is redeployed or OOM-killed is the failure that actually
+        happens in production, and it is the one the probe gate stopped
+        covering: ``_verified`` short-circuits ahead of it. The wave in flight
+        when the worker dies cannot be saved -- every caller is already past
+        the check -- but every wave after it must be gated again.
+        """
+        warm = threading.Event()
+        warm.set()
+        healthy = self._blocking_celery(warm, [[0.4]])
+        embeddings = DelegatedEmbeddings("granite-311m")
+        with patch.dict(
+            "sys.modules", {"application.celery_init": MagicMock(celery=healthy)}
+        ):
+            embeddings.embed_query("warm")
+        assert embeddings._verified is True
+
+        # No cooldown, so anything that gates the second wave can only be the
+        # probe -- which engages only because the failure cleared _verified.
+        with patch(
+            "application.vectorstore.embeddings_delegated._FAILURE_COOLDOWN", 0.0
+        ), patch("application.vectorstore.embeddings_delegated._PROBE_WAIT", 0.05):
+            dying = threading.Event()
+            died = self._blocking_celery(dying, TimeoutError("worker went away"))
+            self._race(died, embeddings, dying)
+            # The wave that was already in flight all dispatched, as it must.
+            assert died.send_task.call_count == 8
+            assert embeddings._verified is False
+
+            again = threading.Event()
+            still_dead = self._blocking_celery(again, TimeoutError("still gone"))
+            values, errors = self._race(still_dead, embeddings, again)
+
+        assert values == []
+        assert len(errors) == 8
+        # One probe pays the timeout; the other seven fail fast.
+        assert still_dead.send_task.call_count == 1
+        assert sum("still unanswered" in str(e) for e in errors) == 7
+
+
+class TestTheResultIsForgotten:
+    """A query vector must not outlive the query that asked for it.
+
+    ``result_expires`` is 7 days and ``embed_texts`` stores its result, but the
+    key is ``celery-task-meta-<uuid>`` -- minted per dispatch, never derived
+    from the text -- so nothing reads it back and a repeated query mints
+    another. Without ``forget()`` every search leaks ~17 KB into the Redis the
+    broker shares for a week.
+    """
+
+    @staticmethod
+    def _celery(side_effect=None, value=None):
+        result = MagicMock()
+        result.get.side_effect = side_effect
+        result.get.return_value = value
+        celery = MagicMock()
+        celery.send_task.return_value = result
+        return celery, result
+
+    def test_a_successful_embed_forgets_its_result(self, not_in_worker):
+        celery, result = self._celery(value=[[0.1, 0.2]])
+        embeddings = DelegatedEmbeddings("granite-311m")
+        with patch.dict("sys.modules", {"application.celery_init": MagicMock(celery=celery)}):
+            assert embeddings.embed_query("q") == [0.1, 0.2]
+        result.forget.assert_called_once()
+
+    def test_a_failed_embed_still_forgets(self, not_in_worker):
+        celery, result = self._celery(side_effect=TimeoutError("no worker"))
+        embeddings = DelegatedEmbeddings("granite-311m")
+        with patch.dict("sys.modules", {"application.celery_init": MagicMock(celery=celery)}):
+            with pytest.raises(RuntimeError):
+                embeddings.embed_query("q")
+        result.forget.assert_called_once()
+
+    def test_a_backend_that_cannot_delete_does_not_fail_the_query(self, not_in_worker):
+        celery, result = self._celery(value=[[0.3, 0.4]])
+        result.forget.side_effect = ConnectionError("backend down")
+        embeddings = DelegatedEmbeddings("granite-311m")
+        with patch.dict("sys.modules", {"application.celery_init": MagicMock(celery=celery)}):
+            assert embeddings.embed_query("q") == [0.3, 0.4]
+
+    def test_forgetting_does_not_mask_the_dispatch_failure(self, not_in_worker):
+        celery, result = self._celery(side_effect=TimeoutError("no worker"))
+        result.forget.side_effect = ConnectionError("backend down")
+        embeddings = DelegatedEmbeddings("granite-311m")
+        with patch.dict("sys.modules", {"application.celery_init": MagicMock(celery=celery)}):
+            with pytest.raises(RuntimeError, match="timed out or failed"):
+                embeddings.embed_query("q")

tests/vectorstore/test_embeddings_local.py

@@ -1,156 +1,393 @@
-from unittest.mock import MagicMock, Mock, patch
+"""Local embeddings run through FastEmbed, configured from the model registry."""
 
+from unittest.mock import MagicMock, patch
+
+import numpy as np
 import pytest
 
+from application.vectorstore import embeddings_local
+from application.vectorstore.embeddings_local import EmbeddingsWrapper
+from application.vectorstore.model_registry import GRANITE_97M, MPNET
 
-@pytest.mark.unit
-class TestEmbeddingsWrapper:
-    @patch("application.vectorstore.embeddings_local.SentenceTransformer")
-    def test_init_success(self, mock_st_cls):
-        mock_model = MagicMock()
-        mock_model._first_module.return_value = MagicMock()
-        mock_model.get_embedding_dimension.return_value = 768
-        mock_st_cls.return_value = mock_model
 
-        from application.vectorstore.embeddings_local import EmbeddingsWrapper
+@pytest.fixture(autouse=True)
+def _clear_registration():
+    """``add_custom_model`` writes to a FastEmbed global; keep tests isolated."""
+    embeddings_local._registered.clear()
+    yield
+    embeddings_local._registered.clear()
 
-        wrapper = EmbeddingsWrapper("test-model")
 
-        mock_st_cls.assert_called_once()
-        assert wrapper.dimension == 768
+@pytest.fixture(autouse=True)
+def _no_hub_reads():
+    """Keep unit tests off the network.
 
-    @patch("application.vectorstore.embeddings_local.SentenceTransformer")
-    def test_init_falls_back_to_legacy_dimension_api(self, mock_st_cls):
-        """sentence-transformers < 5.4 only exposes get_sentence_embedding_dimension."""
-        mock_model = MagicMock()
-        mock_model._first_module.return_value = MagicMock()
-        del mock_model.get_embedding_dimension
-        mock_model.get_sentence_embedding_dimension.return_value = 768
-        mock_st_cls.return_value = mock_model
+    ``_spec_for`` now asks a repository how it pools; without this every test
+    naming an unregistered model would reach the Hugging Face hub. ``None`` is
+    the "declares nothing" answer, which is the behaviour these tests were
+    written against. Tests that exercise the metadata patch it themselves.
+    """
+    with patch.object(embeddings_local, "_read_repo_json", return_value=None):
+        yield
 
-        from application.vectorstore.embeddings_local import EmbeddingsWrapper
 
-        wrapper = EmbeddingsWrapper("test-model")
+@pytest.fixture
+def fake_fastembed():
+    """Patch FastEmbed so no model is downloaded or run."""
+    text_embedding = MagicMock()
+    instance = MagicMock()
+    instance.embed.return_value = iter([np.array([0.1, 0.2, 0.3])])
+    text_embedding.return_value = instance
+    # Registration checks this before calling ``add_custom_model``; an empty
+    # list means "no built-in collides", which is the case for every name in
+    # our registry.
+    text_embedding.list_supported_models.return_value = []
+    with patch("fastembed.TextEmbedding", text_embedding):
+        yield text_embedding, instance
 
-        assert wrapper.dimension == 768
-        mock_model.get_sentence_embedding_dimension.assert_called_once()
 
-    @patch("application.vectorstore.embeddings_local.SentenceTransformer")
-    def test_init_failure(self, mock_st_cls):
-        mock_st_cls.side_effect = Exception("model not found")
+class TestBuiltinModelRegistration:
+    """FastEmbed ships ~30 models of its own and refuses to re-register any of
+    them, so registering unconditionally broke every natively-supported name."""
 
-        from application.vectorstore.embeddings_local import EmbeddingsWrapper
+    def test_builtin_name_is_not_re_registered(self, fake_fastembed):
+        text_embedding, _ = fake_fastembed
+        text_embedding.list_supported_models.return_value = [
+            {"model": "BAAI/bge-small-en-v1.5"}
+        ]
+        EmbeddingsWrapper("BAAI/bge-small-en-v1.5")
+        text_embedding.add_custom_model.assert_not_called()
+        assert text_embedding.call_args.kwargs["model_name"] == "BAAI/bge-small-en-v1.5"
 
-        with pytest.raises(Exception, match="model not found"):
-            EmbeddingsWrapper("bad-model")
+    def test_builtin_match_ignores_case(self, fake_fastembed):
+        text_embedding, _ = fake_fastembed
+        text_embedding.list_supported_models.return_value = [
+            {"model": "baai/BGE-Small-EN-v1.5"}
+        ]
+        EmbeddingsWrapper("BAAI/bge-small-en-v1.5")
+        text_embedding.add_custom_model.assert_not_called()
 
-    @patch("application.vectorstore.embeddings_local.SentenceTransformer")
-    def test_init_none_model(self, mock_st_cls):
-        mock_st_cls.return_value = None
+    def test_unknown_name_is_still_registered(self, fake_fastembed):
+        text_embedding, _ = fake_fastembed
+        text_embedding.list_supported_models.return_value = [
+            {"model": "BAAI/bge-small-en-v1.5"}
+        ]
+        EmbeddingsWrapper("some-org/custom-embedder")
+        text_embedding.add_custom_model.assert_called_once()
 
-        from application.vectorstore.embeddings_local import EmbeddingsWrapper
+    def test_real_fastembed_accepts_its_own_builtin(self):
+        """Runs against the installed FastEmbed, not the MagicMock.
 
-        with pytest.raises((ValueError, AttributeError)):
-            EmbeddingsWrapper("bad-model")
+        The mocked tests above cannot catch this: the failure was
+        ``add_custom_model`` raising, and a MagicMock never raises.
+        """
+        fastembed = pytest.importorskip("fastembed")
+        builtins = [m["model"] for m in fastembed.TextEmbedding.list_supported_models()]
+        assert builtins, "expected FastEmbed to ship built-in models"
+        spec = embeddings_local._spec_for(builtins[0])
+        # Must not raise ValueError("... is already registered ...").
+        embeddings_local._register(spec)
 
-    @patch("application.vectorstore.embeddings_local.SentenceTransformer")
-    def test_init_null_first_module(self, mock_st_cls):
-        mock_model = MagicMock()
-        mock_model._first_module.return_value = None
-        mock_st_cls.return_value = mock_model
 
-        from application.vectorstore.embeddings_local import EmbeddingsWrapper
+class TestRegistryDrivenLoading:
+    def test_registered_model_loads_by_repo_not_by_configured_name(self, fake_fastembed):
+        text_embedding, _ = fake_fastembed
+        wrapper = EmbeddingsWrapper(MPNET.name)
+        assert text_embedding.call_args.kwargs["model_name"] == MPNET.repo
+        assert wrapper.dimension == MPNET.dimension
 
-        with pytest.raises(ValueError, match="failed to load properly"):
-            EmbeddingsWrapper("bad-model")
+    def test_legacy_alias_resolves_to_the_same_model(self, fake_fastembed):
+        text_embedding, _ = fake_fastembed
+        EmbeddingsWrapper("huggingface_sentence-transformers-all-mpnet-base-v2")
+        assert text_embedding.call_args.kwargs["model_name"] == MPNET.repo
 
-    @patch("application.vectorstore.embeddings_local.SentenceTransformer")
-    def test_embed_query(self, mock_st_cls):
-        mock_model = MagicMock()
-        mock_model._first_module.return_value = MagicMock()
-        mock_model.get_sentence_embedding_dimension.return_value = 3
-        mock_model.encode.return_value = MagicMock(tolist=Mock(return_value=[0.1, 0.2, 0.3]))
-        mock_st_cls.return_value = mock_model
+    def test_dimension_comes_from_registry_without_running_the_model(self, fake_fastembed):
+        _, instance = fake_fastembed
+        wrapper = EmbeddingsWrapper(GRANITE_97M.name)
+        assert wrapper.dimension == 384
+        instance.embed.assert_not_called()
 
-        from application.vectorstore.embeddings_local import EmbeddingsWrapper
+    def test_unknown_model_is_treated_as_a_hf_repo(self, fake_fastembed):
+        text_embedding, _ = fake_fastembed
+        wrapper = EmbeddingsWrapper("some-org/custom-embedder")
+        assert text_embedding.call_args.kwargs["model_name"] == "some-org/custom-embedder"
+        # No registry entry means no known width, so it must be probed.
+        assert wrapper.dimension == 3
 
-        wrapper = EmbeddingsWrapper("model")
-        result = wrapper.embed_query("hello world")
+    def test_load_failure_names_the_model_and_the_known_ones(self):
+        with patch("fastembed.TextEmbedding", side_effect=OSError("no such repo")):
+            with pytest.raises(RuntimeError) as excinfo:
+                EmbeddingsWrapper("broken/model")
+        message = str(excinfo.value)
+        assert "broken/model" in message
+        assert MPNET.name in message
 
-        mock_model.encode.assert_called_once_with("hello world")
-        assert result == [0.1, 0.2, 0.3]
 
-    @patch("application.vectorstore.embeddings_local.SentenceTransformer")
-    def test_embed_documents(self, mock_st_cls):
-        mock_model = MagicMock()
-        mock_model._first_module.return_value = MagicMock()
-        mock_model.get_sentence_embedding_dimension.return_value = 3
-        mock_model.encode.return_value = MagicMock(
-            tolist=Mock(return_value=[[0.1, 0.2], [0.3, 0.4]])
+class TestSettingsPassthrough:
+    def test_threads_forwarded_when_configured(self, fake_fastembed):
+        text_embedding, _ = fake_fastembed
+        with patch.object(embeddings_local.settings, "EMBEDDINGS_THREADS", 2, create=True):
+            EmbeddingsWrapper(MPNET.name)
+        assert text_embedding.call_args.kwargs["threads"] == 2
+
+    def test_threads_omitted_when_unset(self, fake_fastembed):
+        text_embedding, _ = fake_fastembed
+        with patch.object(embeddings_local.settings, "EMBEDDINGS_THREADS", None, create=True):
+            EmbeddingsWrapper(MPNET.name)
+        assert "threads" not in text_embedding.call_args.kwargs
+
+    def test_cache_dir_forwarded_when_configured(self, fake_fastembed):
+        text_embedding, _ = fake_fastembed
+        with patch.object(embeddings_local.settings, "EMBEDDINGS_CACHE_DIR", "/models", create=True):
+            EmbeddingsWrapper(MPNET.name)
+        assert text_embedding.call_args.kwargs["cache_dir"] == "/models"
+
+
+class TestEmbedding:
+    def test_embed_documents_returns_plain_lists(self, fake_fastembed):
+        _, instance = fake_fastembed
+        instance.embed.return_value = iter([np.array([1.0, 2.0]), np.array([3.0, 4.0])])
+        wrapper = EmbeddingsWrapper(MPNET.name)
+        assert wrapper.embed_documents(["a", "b"]) == [[1.0, 2.0], [3.0, 4.0]]
+
+    def test_embed_documents_short_circuits_on_empty_input(self, fake_fastembed):
+        _, instance = fake_fastembed
+        wrapper = EmbeddingsWrapper(MPNET.name)
+        instance.embed.reset_mock()
+        assert wrapper.embed_documents([]) == []
+        instance.embed.assert_not_called()
+
+    def test_embed_query_returns_a_single_vector(self, fake_fastembed):
+        _, instance = fake_fastembed
+        instance.embed.return_value = iter([np.array([0.5, 0.6])])
+        wrapper = EmbeddingsWrapper(MPNET.name)
+        assert wrapper.embed_query("hello") == [0.5, 0.6]
+
+    def test_call_dispatches_on_input_type(self, fake_fastembed):
+        _, instance = fake_fastembed
+        wrapper = EmbeddingsWrapper(MPNET.name)
+        instance.embed.return_value = iter([np.array([1.0])])
+        assert wrapper("text") == [1.0]
+        instance.embed.return_value = iter([np.array([1.0]), np.array([2.0])])
+        assert wrapper(["a", "b"]) == [[1.0], [2.0]]
+
+    def test_call_rejects_other_types(self, fake_fastembed):
+        wrapper = EmbeddingsWrapper(MPNET.name)
+        with pytest.raises(ValueError):
+            wrapper(42)
+
+
+class TestRegistrationIsIdempotent:
+    def test_model_registered_once_per_process(self, fake_fastembed):
+        text_embedding, _ = fake_fastembed
+        EmbeddingsWrapper(MPNET.name)
+        EmbeddingsWrapper(MPNET.name)
+        assert text_embedding.add_custom_model.call_count == 1
+
+
+class TestLengthSortedBatching:
+    """Grouping by length is a throughput/memory win, but order is a contract."""
+
+    def _wrapper(self, fake_fastembed, batch_size):
+        _, instance = fake_fastembed
+        wrapper = EmbeddingsWrapper(MPNET.name)
+        instance.embed.side_effect = lambda texts, batch_size=None: iter(
+            [np.array([float(len(t))]) for t in texts]
         )
-        mock_st_cls.return_value = mock_model
+        return wrapper, instance
 
-        from application.vectorstore.embeddings_local import EmbeddingsWrapper
+    def test_output_order_matches_input_order(self, fake_fastembed):
+        with patch.object(embeddings_local.settings, "EMBEDDINGS_MODEL_BATCH_SIZE", 2, create=True):
+            wrapper, _ = self._wrapper(fake_fastembed, 2)
+            texts = ["dddd", "a", "ccc", "bb", "eeeee"]
+            out = wrapper.embed_documents(texts)
+        # Each stub vector encodes its own text length, so a reordered result
+        # is immediately visible.
+        assert out == [[4.0], [1.0], [3.0], [2.0], [5.0]]
 
-        wrapper = EmbeddingsWrapper("model")
-        result = wrapper.embed_documents(["doc1", "doc2"])
+    def test_inputs_are_grouped_by_length_before_batching(self, fake_fastembed):
+        with patch.object(embeddings_local.settings, "EMBEDDINGS_MODEL_BATCH_SIZE", 2, create=True):
+            wrapper, instance = self._wrapper(fake_fastembed, 2)
+            wrapper.embed_documents(["dddd", "a", "ccc", "bb", "eeeee"])
+        sent = instance.embed.call_args.args[0]
+        assert [len(t) for t in sent] == [1, 2, 3, 4, 5]
 
-        mock_model.encode.assert_called_with(["doc1", "doc2"])
-        assert result == [[0.1, 0.2], [0.3, 0.4]]
+    def test_single_batch_is_not_reordered(self, fake_fastembed):
+        with patch.object(embeddings_local.settings, "EMBEDDINGS_MODEL_BATCH_SIZE", 32, create=True):
+            wrapper, instance = self._wrapper(fake_fastembed, 32)
+            texts = ["dddd", "a", "ccc"]
+            out = wrapper.embed_documents(texts)
+        assert instance.embed.call_args.args[0] == texts
+        assert out == [[4.0], [1.0], [3.0]]
 
-    @patch("application.vectorstore.embeddings_local.SentenceTransformer")
-    def test_call_with_string(self, mock_st_cls):
-        mock_model = MagicMock()
-        mock_model._first_module.return_value = MagicMock()
-        mock_model.get_sentence_embedding_dimension.return_value = 3
-        mock_model.encode.return_value = MagicMock(tolist=Mock(return_value=[0.1]))
-        mock_st_cls.return_value = mock_model
+    def test_duplicate_texts_are_handled(self, fake_fastembed):
+        with patch.object(embeddings_local.settings, "EMBEDDINGS_MODEL_BATCH_SIZE", 2, create=True):
+            wrapper, _ = self._wrapper(fake_fastembed, 2)
+            out = wrapper.embed_documents(["aa", "b", "aa", "ccc"])
+        assert out == [[2.0], [1.0], [2.0], [3.0]]
 
-        from application.vectorstore.embeddings_local import EmbeddingsWrapper
 
-        wrapper = EmbeddingsWrapper("model")
-        result = wrapper("hello")
-        assert result == [0.1]
+class TestTokenizerPadding:
+    """A fixed padding width in ``tokenizer.json`` makes mixed batches ragged.
 
-    @patch("application.vectorstore.embeddings_local.SentenceTransformer")
-    def test_call_with_list(self, mock_st_cls):
-        mock_model = MagicMock()
-        mock_model._first_module.return_value = MagicMock()
-        mock_model.get_sentence_embedding_dimension.return_value = 3
-        mock_model.encode.return_value = MagicMock(
-            tolist=Mock(return_value=[[0.1], [0.2]])
+    FastEmbed calls ``enable_padding`` only when the tokenizer declares none,
+    so mpnet's fixed ``length: 128`` survives loading. Any batch mixing an
+    input longer than 128 tokens with a shorter one then produces rows of
+    different widths and ONNX rejects the tensor.
+    """
+
+    def _tokenizer(self, padding):
+        tokenizer = MagicMock()
+        tokenizer.padding = padding
+        return tokenizer
+
+    def test_fixed_width_padding_is_reset_to_batch_longest(self, fake_fastembed):
+        _, instance = fake_fastembed
+        tokenizer = self._tokenizer(
+            {
+                "length": 128,
+                "pad_id": 1,
+                "pad_token": "<pad>",
+                "pad_type_id": 0,
+                "direction": "right",
+                "pad_to_multiple_of": None,
+            }
         )
-        mock_st_cls.return_value = mock_model
+        instance.model.tokenizer = tokenizer
 
-        from application.vectorstore.embeddings_local import EmbeddingsWrapper
+        EmbeddingsWrapper(MPNET.name)
 
-        wrapper = EmbeddingsWrapper("model")
-        result = wrapper(["a", "b"])
-        assert result == [[0.1], [0.2]]
+        kwargs = tokenizer.enable_padding.call_args.kwargs
+        assert kwargs["length"] is None, "padding must follow the longest input"
+        # The model's own pad token must survive the reset.
+        assert kwargs["pad_id"] == 1
+        assert kwargs["pad_token"] == "<pad>"
 
-    @patch("application.vectorstore.embeddings_local.SentenceTransformer")
-    def test_call_with_invalid_type(self, mock_st_cls):
-        mock_model = MagicMock()
-        mock_model._first_module.return_value = MagicMock()
-        mock_model.get_sentence_embedding_dimension.return_value = 3
-        mock_st_cls.return_value = mock_model
+    def test_dynamic_padding_is_left_alone(self, fake_fastembed):
+        _, instance = fake_fastembed
+        tokenizer = self._tokenizer({"length": None, "pad_id": 0, "pad_token": "<pad>"})
+        instance.model.tokenizer = tokenizer
 
-        from application.vectorstore.embeddings_local import EmbeddingsWrapper
+        EmbeddingsWrapper(GRANITE_97M.name)
 
-        wrapper = EmbeddingsWrapper("model")
-        with pytest.raises(ValueError, match="Input must be a string or a list"):
-            wrapper(123)
+        tokenizer.enable_padding.assert_not_called()
 
-    @patch("application.vectorstore.embeddings_local.SentenceTransformer")
-    def test_trust_remote_code_default(self, mock_st_cls):
-        mock_model = MagicMock()
-        mock_model._first_module.return_value = MagicMock()
-        mock_model.get_sentence_embedding_dimension.return_value = 768
-        mock_st_cls.return_value = mock_model
+    def test_tokenizer_that_cannot_be_reached_is_not_fatal(self, fake_fastembed):
+        _, instance = fake_fastembed
+        instance.model = None
+        EmbeddingsWrapper(GRANITE_97M.name)
 
-        from application.vectorstore.embeddings_local import EmbeddingsWrapper
 
-        EmbeddingsWrapper("model")
+def _repo_json(pooling_file, modules_file):
+    """Stub ``_read_repo_json`` returning canned repository metadata."""
 
-        call_kwargs = mock_st_cls.call_args[1]
-        assert call_kwargs["trust_remote_code"] is True
+    def read(repo, filename):
+        return pooling_file if filename == embeddings_local._POOLING_CONFIG else modules_file
+
+    return read
+
+
+class TestPoolingReadFromTheRepository:
+    """A model's pooling is a fact its repository states, not a default.
+
+    Assuming mean pooling for a CLS model returns vectors at cosine ~0.95 to
+    the correct ones: no error, no dimension mismatch, just quietly worse
+    retrieval. These cover the shapes seen on the hub.
+    """
+
+    def test_cls_pooling_is_read_rather_than_assumed(self):
+        with patch.object(
+            embeddings_local,
+            "_read_repo_json",
+            _repo_json(
+                {"pooling_mode_cls_token": True, "word_embedding_dimension": 384},
+                [{"type": "sentence_transformers.models.Transformer"},
+                 {"type": "sentence_transformers.models.Pooling"},
+                 {"type": "sentence_transformers.models.Normalize"}],
+            ),
+        ):
+            spec = embeddings_local._spec_for("BAAI/bge-small-en-v1.5")
+        assert spec.pooling == "cls"
+        assert spec.normalize is True
+        # Declared width, so no probe run is needed to learn it.
+        assert spec.dimension == 384
+
+    def test_missing_normalize_module_means_unnormalised(self):
+        """multi-qa-mpnet-base-dot-v1 is trained on unnormalised vectors."""
+        with patch.object(
+            embeddings_local,
+            "_read_repo_json",
+            _repo_json(
+                {"pooling_mode_cls_token": True, "word_embedding_dimension": 768},
+                [{"type": "sentence_transformers.models.Transformer"},
+                 {"type": "sentence_transformers.models.Pooling"}],
+            ),
+        ):
+            spec = embeddings_local._spec_for("sentence-transformers/multi-qa-mpnet-base-dot-v1")
+        assert spec.pooling == "cls"
+        assert spec.normalize is False
+
+    def test_dense_projection_head_is_refused(self):
+        """FastEmbed would skip the projection and emit the wrong vectors."""
+        with patch.object(
+            embeddings_local,
+            "_read_repo_json",
+            _repo_json(
+                {"pooling_mode_cls_token": True, "word_embedding_dimension": 768},
+                [{"type": "sentence_transformers.models.Transformer"},
+                 {"type": "sentence_transformers.models.Pooling"},
+                 {"type": "sentence_transformers.models.Dense"},
+                 {"type": "sentence_transformers.models.Normalize"}],
+            ),
+        ):
+            with pytest.raises(RuntimeError) as excinfo:
+                embeddings_local._spec_for("sentence-transformers/LaBSE")
+        message = str(excinfo.value)
+        assert "LaBSE" in message
+        assert "Dense" in message
+
+    def test_unsupported_pooling_mode_falls_back_rather_than_lying(self):
+        with patch.object(
+            embeddings_local,
+            "_read_repo_json",
+            _repo_json({"pooling_mode_max_tokens": True}, []),
+        ):
+            spec = embeddings_local._spec_for("some-org/max-pooled")
+        assert spec.pooling == embeddings_local._FALLBACK_POOLING
+        assert spec.dimension == 0
+
+    def test_repository_without_metadata_keeps_the_assumption(self):
+        spec = embeddings_local._spec_for("some-org/plain-onnx-export")
+        assert spec.pooling == embeddings_local._FALLBACK_POOLING
+        assert spec.normalize is True
+        assert spec.dimension == 0
+
+    def test_registry_wins_over_the_repository(self):
+        """A described model is never re-read; the registry is the answer."""
+        read = MagicMock()
+        with patch.object(embeddings_local, "_read_repo_json", read):
+            spec = embeddings_local._spec_for(MPNET.name)
+        assert spec is MPNET
+        read.assert_not_called()
+
+
+class TestPoolingOverrides:
+    def test_settings_override_what_the_repository_declares(self):
+        with patch.object(
+            embeddings_local,
+            "_read_repo_json",
+            _repo_json(
+                {"pooling_mode_mean_tokens": True, "word_embedding_dimension": 768},
+                [{"type": "sentence_transformers.models.Normalize"}],
+            ),
+        ):
+            with patch.object(embeddings_local.settings, "EMBEDDINGS_POOLING", "cls"), \
+                 patch.object(embeddings_local.settings, "EMBEDDINGS_NORMALIZE", False):
+                spec = embeddings_local._spec_for("some-org/mislabelled")
+        assert spec.pooling == "cls"
+        assert spec.normalize is False
+
+    def test_a_meaningless_override_is_ignored(self):
+        with patch.object(embeddings_local.settings, "EMBEDDINGS_POOLING", "banana"):
+            spec = embeddings_local._spec_for("some-org/plain-onnx-export")
+        assert spec.pooling == embeddings_local._FALLBACK_POOLING

tests/vectorstore/test_faiss.py

@@ -5,11 +5,13 @@ than asserting that calls were forwarded to a mock.
 """
 
 import json
-from unittest.mock import Mock, patch
+from types import SimpleNamespace
+from unittest.mock import MagicMock, Mock, patch
 
 import pytest
 
 from application.storage.local import LocalStorage
+from application.vectorstore.faiss import FaissStore
 
 
 class _FakeEmbeddings:
@@ -185,15 +187,20 @@ class TestFaissStoreAssertEmbeddingDimensions:
             with pytest.raises(ValueError, match="Embedding dimension mismatch"):
                 populated.assert_embedding_dimensions(Mock(dimension=768))
 
-    def test_missing_dimension_attr_raises(self, populated):
+    def test_unknown_dimension_defers_rather_than_raising(self, populated):
+        """A remote model reports no width until its first call.
+
+        Refusing to open the index in that window would break startup for a
+        perfectly valid remote configuration, so an unknown width is deferred,
+        not treated as a mismatch.
+        """
         with patch("application.vectorstore.faiss.settings") as mock_settings:
             mock_settings.EMBEDDINGS_NAME = (
                 "huggingface_sentence-transformers/all-mpnet-base-v2"
             )
             embeddings = Mock()
-            del embeddings.dimension
-            with pytest.raises(AttributeError, match="'dimension' attribute not found"):
-                populated.assert_embedding_dimensions(embeddings)
+            embeddings.dimension = None
+            assert populated.assert_embedding_dimensions(embeddings) is None
 
     def test_dimension_match_passes(self, populated):
         with patch("application.vectorstore.faiss.settings") as mock_settings:
@@ -202,10 +209,22 @@ class TestFaissStoreAssertEmbeddingDimensions:
             )
             assert populated.assert_embedding_dimensions(Mock(dimension=3)) is None
 
-    def test_non_huggingface_skips_dimension_check(self, populated):
+    def test_mismatch_is_caught_for_every_model_not_just_mpnet(self, populated):
+        """The check used to run only when EMBEDDINGS_NAME was mpnet.
+
+        That skipped exactly the case it exists for: an index built with one
+        model being opened under a different one.
+        """
         with patch("application.vectorstore.faiss.settings") as mock_settings:
             mock_settings.EMBEDDINGS_NAME = "openai_text-embedding-ada-002"
-            assert populated.assert_embedding_dimensions(Mock(dimension=1536)) is None
+            with pytest.raises(ValueError, match="Embedding dimension mismatch"):
+                populated.assert_embedding_dimensions(Mock(dimension=1536))
+
+    def test_mismatch_message_points_at_the_reembed_script(self, populated):
+        with patch("application.vectorstore.faiss.settings") as mock_settings:
+            mock_settings.EMBEDDINGS_NAME = "granite-311m"
+            with pytest.raises(ValueError, match="application.scripts.reembed"):
+                populated.assert_embedding_dimensions(Mock(dimension=768))
 
 
 @pytest.mark.unit
@@ -226,3 +245,45 @@ class TestGetVectorstore:
 
         with pytest.raises(ValueError, match="Invalid source_id path"):
             get_vectorstore(bad)
+
+
+class TestBuildFromDocumentsBatching:
+    """The full rebuild path: honour caller ids and embed in batches."""
+
+    def _docs(self, n):
+        return [
+            SimpleNamespace(page_content=f"chunk {i}", metadata={"i": i})
+            for i in range(n)
+        ]
+
+    def _store(self, embed):
+        store = FaissStore.__new__(FaissStore)
+        store.embeddings = MagicMock()
+        store.embeddings.embed_documents.side_effect = embed
+        store.documents = {}
+        store.index_to_docstore_id = {}
+        store.index = None
+        return store
+
+    def test_supplied_ids_are_used(self):
+        store = self._store(lambda texts: [[float(len(texts))] * 2 for _ in texts])
+        store._build_from_documents(self._docs(3), ids=["a", "b", "c"])
+        assert list(store.documents) == ["a", "b", "c"]
+
+    def test_embedding_is_split_into_batches(self):
+        sizes = []
+
+        def embed(texts):
+            sizes.append(len(texts))
+            return [[1.0, 2.0] for _ in texts]
+
+        store = self._store(embed)
+        store._build_from_documents(self._docs(5), batch_size=2)
+        assert sizes == [2, 2, 1]
+        assert len(store.index_to_docstore_id) == 5
+
+    def test_defaults_are_unchanged(self):
+        store = self._store(lambda texts: [[1.0, 2.0] for _ in texts])
+        store._build_from_documents(self._docs(3))
+        assert len(store.documents) == 3
+        assert all(len(k) == 36 for k in store.documents), "uuid4 ids by default"

tests/vectorstore/test_model_registry.py

@@ -0,0 +1,117 @@
+"""The registry is the single source of truth for embedding-model facts."""
+
+import os
+
+import pytest
+
+from application.vectorstore import model_registry as reg
+
+
+class TestResolve:
+    def test_resolves_canonical_name(self):
+        assert reg.resolve(reg.MPNET.name) is reg.MPNET
+
+    @pytest.mark.parametrize(
+        "alias",
+        [
+            "huggingface_sentence-transformers-all-mpnet-base-v2",
+            "sentence-transformers/all-mpnet-base-v2",
+            "all-mpnet-base-v2",
+        ],
+    )
+    def test_resolves_legacy_spellings_of_mpnet(self, alias):
+        """Every spelling the old factory dict accepted must still work."""
+        assert reg.resolve(alias) is reg.MPNET
+
+    def test_resolution_is_case_insensitive_and_trims(self):
+        assert reg.resolve("  GRANITE-311M  ") is reg.GRANITE_311M
+
+    def test_unknown_name_is_none_not_an_error(self):
+        """Unknown names are a valid configuration: an arbitrary HF repo."""
+        assert reg.resolve("some-org/some-model") is None
+
+    def test_empty_and_none_resolve_to_none(self):
+        assert reg.resolve(None) is None
+        assert reg.resolve("") is None
+
+
+class TestModelFacts:
+    def test_granite_311m_matches_the_existing_column_width(self):
+        """768 is what makes granite a drop-in for an mpnet index."""
+        assert reg.GRANITE_311M.dimension == reg.MPNET.dimension == 768
+
+    def test_granite_97m_is_narrower(self):
+        assert reg.GRANITE_97M.dimension == 384
+
+    def test_pooling_and_normalisation_are_recorded(self):
+        assert reg.MPNET.pooling == "mean"
+        assert reg.GRANITE_311M.pooling == "cls"
+        assert all(m.normalize for m in reg.MODELS)
+
+    def test_granite_context_is_much_wider_than_mpnet(self):
+        assert reg.GRANITE_311M.max_input_tokens == 32768
+        assert reg.MPNET.max_input_tokens == 384
+
+    def test_local_runners_carry_a_repo_and_onnx_file(self):
+        for model in reg.MODELS:
+            if model.provider == "fastembed":
+                assert model.repo, f"{model.name} has no repo"
+                assert model.onnx_file, f"{model.name} has no onnx_file"
+
+    def test_openai_model_needs_no_local_artifacts(self):
+        assert reg.OPENAI_ADA_002.provider == "openai"
+        assert reg.OPENAI_ADA_002.repo is None
+
+
+class TestHelpers:
+    def test_dimension_for_known_and_unknown(self):
+        assert reg.dimension_for(reg.GRANITE_97M.name) == 384
+        assert reg.dimension_for("nope/nope") is None
+
+    def test_max_input_tokens_for_known_and_unknown(self):
+        assert reg.max_input_tokens_for("granite-311m") == 32768
+        assert reg.max_input_tokens_for("nope/nope") is None
+
+    def test_known_names_lists_canonical_spellings(self):
+        names = reg.known_names()
+        assert reg.MPNET.name in names
+        assert reg.GRANITE_311M.name in names
+
+    def test_defaults_point_at_the_intended_models(self):
+        """Existing installs stay on mpnet; new installs get granite."""
+        assert reg.DEFAULT_LEGACY == reg.MPNET.name
+        assert reg.DEFAULT_NEW_INSTALL == reg.GRANITE_311M.name
+
+    def test_no_alias_collisions_between_models(self):
+        seen = {}
+        for model in reg.MODELS:
+            for key in (model.name, *model.aliases):
+                assert key.lower() not in seen, f"{key} claimed twice"
+                seen[key.lower()] = model
+
+
+class TestRegistryMatchesTheHub:
+    """Every registry entry restates facts the model's repository already holds.
+
+    Restating them is what makes an offline install work, but a restatement can
+    drift from its source and nothing else would notice: wrong pooling is not a
+    crash, only worse retrieval. Opt in with ``DOCSGPT_HUB_TESTS=1``; needs
+    network.
+    """
+
+    @pytest.mark.skipif(
+        os.environ.get("DOCSGPT_HUB_TESTS") != "1",
+        reason="set DOCSGPT_HUB_TESTS=1 to check the registry against the hub",
+    )
+    @pytest.mark.parametrize(
+        "model", [m for m in reg.MODELS if m.provider == "fastembed"]
+    )
+    def test_entry_matches_repository_metadata(self, model):
+        from application.vectorstore.embeddings_local import _describe_from_repo
+
+        described = _describe_from_repo(model.repo)
+        assert described is not None, f"{model.repo} declares no pooling metadata"
+        assert described.pooling == model.pooling
+        assert described.normalize == model.normalize
+        if described.dimension:
+            assert described.dimension == model.dimension

tests/vectorstore/test_pgvector_live_schema.py

@@ -103,9 +103,20 @@ def live_dsn(postgresql, monkeypatch):
 
 @pytest.fixture
 def stub_embeddings():
+    """Stand in for the configured model everywhere its width is read.
+
+    ``ensure_vector_schema`` takes the width from the registry rather than by
+    constructing the model, so patching only the constructors would leave the
+    boot hook sizing the table from whatever EMBEDDINGS_NAME happens to be.
+    """
     stub = _StubEmbeddings()
     with patch(
         "application.vectorstore.base.get_embeddings", return_value=stub
+    ), patch(
+        "application.vectorstore.base.build_local_embeddings", return_value=stub
+    ), patch(
+        "application.vectorstore.model_registry.dimension_for",
+        return_value=STUB_DIM,
     ), patch(
         "application.vectorstore.base.BaseVectorStore._get_embeddings",
         return_value=stub,
@@ -140,6 +151,9 @@ class TestBootHookCreatesTheSchema:
         wide = _WideStubEmbeddings()
         with patch(
             "application.vectorstore.base.get_embeddings", return_value=wide
+        ), patch(
+            "application.vectorstore.model_registry.dimension_for",
+            return_value=wide.dimension,
         ):
             with pytest.raises(RuntimeError) as excinfo:
                 ensure_vector_schema()

tests/vectorstore/test_remote_embeddings_truncation.py

@@ -83,3 +83,115 @@ def test_query_path_is_truncated(monkeypatch):
 
     sent = captured["payload"]["input"]
     assert sent == enc.decode(enc.encode(long_text)[:10])
+
+
+class TestInputLimitResolution:
+    """The cap falls back to the model's own context window."""
+
+    def _remote(self, model_name):
+        from application.vectorstore.base import RemoteEmbeddings
+
+        return RemoteEmbeddings(
+            api_url="http://embeddings", model_name=model_name, api_key=None
+        )
+
+    def test_explicit_setting_wins(self, monkeypatch):
+        from application.vectorstore import base
+
+        monkeypatch.setattr(base.settings, "EMBEDDINGS_MAX_INPUT_TOKENS", 123)
+        assert self._remote("granite-311m")._resolve_input_limit() == 123
+
+    def test_registered_model_supplies_its_own_ceiling(self, monkeypatch):
+        from application.vectorstore import base
+
+        monkeypatch.setattr(base.settings, "EMBEDDINGS_MAX_INPUT_TOKENS", None)
+        monkeypatch.setattr(base, "_embeddings_name_is_explicit", lambda: True)
+        assert self._remote("granite-311m")._resolve_input_limit() == 32768
+        # mpnet genuinely stops at 384; sending more is paid for and discarded.
+        assert self._remote("all-mpnet-base-v2")._resolve_input_limit() == 384
+
+    def test_default_model_name_lends_the_server_no_ceiling(self, monkeypatch):
+        """An unset EMBEDDINGS_NAME describes nothing about the remote server.
+
+        The name is only forwarded as the ``model`` field. Letting the
+        settings default contribute mpnet's 384-token window would clip every
+        chunk on a server that may well serve a 32k-context model.
+        """
+        from application.vectorstore import base
+
+        monkeypatch.setattr(base.settings, "EMBEDDINGS_MAX_INPUT_TOKENS", None)
+        monkeypatch.setattr(base, "_embeddings_name_is_explicit", lambda: False)
+        assert self._remote("all-mpnet-base-v2")._resolve_input_limit() is None
+
+    def test_explicit_setting_still_wins_over_an_unset_name(self, monkeypatch):
+        from application.vectorstore import base
+
+        monkeypatch.setattr(base.settings, "EMBEDDINGS_MAX_INPUT_TOKENS", 512)
+        monkeypatch.setattr(base, "_embeddings_name_is_explicit", lambda: False)
+        assert self._remote("all-mpnet-base-v2")._resolve_input_limit() == 512
+
+    def test_unknown_model_stays_unlimited(self, monkeypatch):
+        from application.vectorstore import base
+
+        monkeypatch.setattr(base.settings, "EMBEDDINGS_MAX_INPUT_TOKENS", None)
+        assert self._remote("some-org/mystery")._resolve_input_limit() is None
+
+    def test_non_positive_setting_falls_through_to_the_registry(self, monkeypatch):
+        from application.vectorstore import base
+
+        monkeypatch.setattr(base.settings, "EMBEDDINGS_MAX_INPUT_TOKENS", 0)
+        monkeypatch.setattr(base, "_embeddings_name_is_explicit", lambda: True)
+        assert self._remote("granite-97m")._resolve_input_limit() == 32768
+
+    def test_dimension_is_taken_from_the_registry(self):
+        assert self._remote("granite-97m").dimension == 384
+        assert self._remote("granite-311m").dimension == 768
+
+    def test_unknown_model_dimension_is_probed_not_assumed(self):
+        """The old hardcoded 768 made the probe below unreachable."""
+        assert self._remote("some-org/mystery").dimension is None
+
+
+class TestEmbeddingsNameIsExplicit:
+    """Which names count as chosen decides whether a remote server inherits a
+    context window it may not have.
+
+    The tests above monkeypatch the predicate, so they cannot see it being
+    wrong. These drive the real one.
+    """
+
+    def _default(self):
+        from application.core.settings import Settings
+
+        return Settings.model_fields["EMBEDDINGS_NAME"].default
+
+    def test_the_default_name_is_not_a_choice(self, monkeypatch):
+        """Every setup script has always written this value unconditionally.
+
+        Reading it as deliberate lends the server mpnet's 384-token window and
+        clips ~80% off every chunk, silently, on upgrade. ``model_fields_set``
+        could not tell the difference: pydantic marks a field set for anything
+        that reached it, ``.env`` included.
+        """
+        monkeypatch.setattr(base.settings, "EMBEDDINGS_NAME", self._default())
+        assert base._embeddings_name_is_explicit() is False
+
+    def test_a_different_name_is_a_choice(self, monkeypatch):
+        monkeypatch.setattr(
+            base.settings,
+            "EMBEDDINGS_NAME",
+            "ibm-granite/granite-embedding-311m-multilingual-r2",
+        )
+        assert base._embeddings_name_is_explicit() is True
+
+    def test_dotenv_written_default_does_not_clip(self, monkeypatch):
+        """End to end: the common upgrade path must send the full chunk."""
+        monkeypatch.setattr(base.settings, "EMBEDDINGS_MAX_INPUT_TOKENS", None)
+        monkeypatch.setattr(base.settings, "EMBEDDINGS_NAME", self._default())
+        captured = _capture_post(monkeypatch)
+
+        long_text = " ".join(["word"] * 1000)
+        emb = RemoteEmbeddings(api_url="http://embeddings", model_name=self._default())
+        emb.embed_documents([long_text])
+
+        assert captured["payload"]["input"][0] == long_text

tests/worker/test_attachment_worker.py

@@ -326,3 +326,133 @@ class TestAttachmentZipBombGuard:
 
         # BadZipFile → return quietly; the format parser surfaces a clean error.
         worker._reject_attachment_zip_bomb(str(path))
+
+
+@pytest.mark.unit
+class TestAttachmentTypeGuard:
+    """A binary with no parser must fail rather than be read as text.
+
+    ``SimpleDirectoryReader`` falls through to a plain-text ``open()`` for a
+    suffix it has no parser for, which is how a phone-uploaded ``.mp4`` was
+    once stored as 100k tokens of binary with ``extraction.status == "ok"``.
+    The same fallthrough is what makes a ``.py`` or ``.log`` attachment work,
+    so the guard admits those on content.
+    """
+
+    def test_binary_without_a_parser_fails_instead_of_being_read_as_text(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch, tmp_path
+    ):
+        from application import worker
+
+        local_path = tmp_path / "clip.mp4"
+        local_path.write_bytes(b"\x00\x00\x00\x18ftypisom\x00\x00\x02\x00isomavc1")
+
+        events = []
+        fake_storage = MagicMock(name="storage")
+        fake_storage.process_file.side_effect = lambda path, callback: callback(
+            str(local_path)
+        )
+        monkeypatch.setattr(worker.StorageCreator, "get_storage", lambda: fake_storage)
+        monkeypatch.setattr(
+            worker,
+            "get_default_file_extractor",
+            lambda ocr_enabled=False, pdf_text_fast_path=False: {},
+        )
+        monkeypatch.setattr(
+            worker,
+            "publish_user_event",
+            lambda user, name, payload, **kwargs: events.append((name, payload)),
+        )
+
+        file_info = {
+            "filename": "clip.mp4",
+            "attachment_id": "507f1f77bcf86cd799439012",
+            "path": "uploads/user1/attachments/clip.mp4",
+            "metadata": {"source": "chat"},
+        }
+
+        with pytest.raises(
+            worker.AttachmentRejectedError, match=r"Unsupported file type: \.mp4"
+        ):
+            worker.attachment_worker(task_self, file_info, "user1")
+
+        failed = [payload for name, payload in events if name == "attachment.failed"]
+        assert failed and failed[0]["error"] == "Unsupported file type: .mp4"
+
+    def test_suffix_the_loaded_extractor_cannot_parse_is_refused(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch, tmp_path
+    ):
+        """The guard judges against the parser table actually loaded.
+
+        Without docling the fallback extractor has no .webp parser, so a
+        .webp the route admitted on its name would otherwise be opened as
+        plain text here.
+        """
+        from application import worker
+
+        local_path = tmp_path / "scan.webp"
+        local_path.write_bytes(b"RIFF\x00\x00\x00\x00WEBPVP8 " + bytes(range(256)))
+
+        events = []
+        fake_storage = MagicMock(name="storage")
+        fake_storage.process_file.side_effect = lambda path, callback: callback(
+            str(local_path)
+        )
+        monkeypatch.setattr(worker.StorageCreator, "get_storage", lambda: fake_storage)
+        # The docling-less fallback table: images are handled, .webp is not.
+        monkeypatch.setattr(
+            worker,
+            "get_default_file_extractor",
+            lambda ocr_enabled=False, pdf_text_fast_path=False: {".png": object()},
+        )
+        monkeypatch.setattr(
+            worker,
+            "publish_user_event",
+            lambda user, name, payload, **kwargs: events.append((name, payload)),
+        )
+
+        file_info = {
+            "filename": "scan.webp",
+            "attachment_id": "507f1f77bcf86cd799439014",
+            "path": "uploads/user1/attachments/scan.webp",
+            "metadata": {"source": "chat"},
+        }
+
+        with pytest.raises(
+            worker.AttachmentRejectedError, match=r"Unsupported file type: \.webp"
+        ):
+            worker.attachment_worker(task_self, file_info, "user1")
+
+        failed = [payload for name, payload in events if name == "attachment.failed"]
+        assert failed and failed[0]["error"] == "Unsupported file type: .webp"
+
+    def test_text_without_a_parser_is_parsed(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch, tmp_path
+    ):
+        from application import worker
+
+        local_path = tmp_path / "server.log"
+        local_path.write_text("2026-09-02 ERROR boom\n", encoding="utf-8")
+
+        fake_storage = MagicMock(name="storage")
+        fake_storage.process_file.side_effect = lambda path, callback: callback(
+            str(local_path)
+        )
+        monkeypatch.setattr(worker.StorageCreator, "get_storage", lambda: fake_storage)
+        monkeypatch.setattr(
+            worker,
+            "get_default_file_extractor",
+            lambda ocr_enabled=False, pdf_text_fast_path=False: {},
+        )
+
+        file_info = {
+            "filename": "server.log",
+            "attachment_id": "507f1f77bcf86cd799439013",
+            "path": "uploads/user1/attachments/server.log",
+            "metadata": {"source": "chat"},
+        }
+
+        result = worker.attachment_worker(task_self, file_info, "user1")
+
+        assert result["filename"] == "server.log"
+        assert result["token_count"] > 0

tests/worker/test_reembed_wiki_page.py

@@ -100,6 +100,46 @@ class TestReembedWikiPageWorker:
         )
         assert result == {"status": "embedded", "added": 2, "deleted": 3}
 
+    def test_a_reembed_stamps_the_model_on_the_source(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        """Wiki sources are created with ``model`` NULL, which the boot mismatch
+        check reads as the legacy model and reports as stale on every startup.
+        Stamping here heals a source created before it was recorded.
+        """
+        from application import worker
+        from application.core.settings import settings
+
+        source_id = _seed_source(pg_conn)
+        assert SourcesRepository(pg_conn).get_any(source_id, "alice")["model"] is None
+
+        _patch_store(monkeypatch, MagicMock(name="vector_store"))
+        _patch_repo(monkeypatch, {"content": "body", "title": "T"})
+        _patch_chunker(monkeypatch, [Document(text="c1")])
+
+        worker.reembed_wiki_page_worker(
+            task_self, source_id, "p.md", "hash-1", "alice"
+        )
+
+        row = SourcesRepository(pg_conn).get_any(source_id, "alice")
+        assert row["model"] == settings.EMBEDDINGS_NAME
+
+    def test_a_purge_leaves_the_model_alone(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        """Deleting a page embeds nothing, so it claims nothing about the model."""
+        from application import worker
+
+        source_id = _seed_source(pg_conn)
+        _patch_store(monkeypatch, MagicMock(name="vector_store"))
+        _patch_repo(monkeypatch, None)
+
+        worker.reembed_wiki_page_worker(
+            task_self, source_id, "gone.md", "hash-x", "alice"
+        )
+
+        assert SourcesRepository(pg_conn).get_any(source_id, "alice")["model"] is None
+
     def test_page_missing_purges(
         self, pg_conn, patch_worker_db, task_self, monkeypatch
     ):