feat: durability and idempotency keys (#2450)

* feat: durability and idempotency keys

* feat: more durable frontend

* fix: tests

* fix: mini issues

* fix: better json validation

* fix: tests

tests/worker/test_agent_workers.py

@@ -61,18 +61,34 @@ class TestAgentWebhookWorker:
         assert str(captured["config"]["id"]) == agent_id
         assert captured["input"] == '{"event": "ping"}'
 
-    def test_missing_agent_returns_error(
+    def test_missing_agent_raises(
         self, pg_conn, patch_worker_db, task_self, monkeypatch
     ):
         from application import worker
 
-        # Run with an id that exists in no shape the resolver understands;
-        # ``looks_like_uuid`` is False so the UUID branch is skipped and
-        # the legacy lookup returns None. Task must yield a clean error
-        # result rather than raising.
         monkeypatch.setattr(worker, "run_agent_logic", lambda *a, **k: {})
-        result = worker.agent_webhook_worker(task_self, "no-such-agent", {})
-        assert result["status"] == "error"
+        with pytest.raises(ValueError, match="not found"):
+            worker.agent_webhook_worker(task_self, "no-such-agent", {})
+
+    def test_run_agent_logic_failure_propagates(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+        from application.storage.db.repositories.agents import AgentsRepository
+
+        agent = AgentsRepository(pg_conn).create(
+            user_id="alice", name="hook-agent", status="active",
+            agent_type="classic", retriever="classic", chunks=2, key="sk-test-123",
+        )
+        agent_id = str(agent["id"])
+
+        def _boom(*a, **k):
+            raise RuntimeError("LLM exploded")
+
+        monkeypatch.setattr(worker, "run_agent_logic", _boom)
+
+        with pytest.raises(RuntimeError, match="LLM exploded"):
+            worker.agent_webhook_worker(task_self, agent_id, {"event": "ping"})
 
 
 @pytest.mark.unit

tests/worker/test_ingest_checkpoint.py

@@ -0,0 +1,531 @@
+"""Tests for the ingest chunk checkpoint and heartbeat.
+
+* ``embed_and_store_documents`` writes/reads ``ingest_chunk_progress``;
+  fresh runs end with ``embedded_chunks=N`` and resumes seeded at
+  ``last_index=k`` embed only chunks ``k+1..N-1``.
+* ``_ingest_heartbeat_loop`` is the daemon body that bumps ``last_updated``.
+"""
+
+from __future__ import annotations
+
+import threading
+import uuid
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from typing import Iterator
+from unittest.mock import MagicMock
+
+import pytest
+from sqlalchemy import Connection, text
+
+
+@dataclass
+class _LCDoc:
+    """Tiny stand-in for a LangChain document — only the fields the loop reads."""
+    page_content: str
+    metadata: dict = field(default_factory=dict)
+
+
+@pytest.fixture
+def patch_pipeline_db(pg_conn, monkeypatch):
+    """Make ``db_session`` inside the embedding pipeline yield ``pg_conn``."""
+
+    @contextmanager
+    def _use_pg_conn() -> Iterator[Connection]:
+        yield pg_conn
+
+    monkeypatch.setattr(
+        "application.parser.embedding_pipeline.db_session", _use_pg_conn
+    )
+
+
+@pytest.fixture
+def faiss_settings(monkeypatch):
+    """Force the embed pipeline down the faiss branch with a stub vector store."""
+    from application.parser import embedding_pipeline as ep
+
+    monkeypatch.setattr(ep.settings, "VECTOR_STORE", "faiss", raising=False)
+
+    fake_store = MagicMock(name="vector_store")
+    monkeypatch.setattr(
+        ep.VectorCreator, "create_vectorstore", lambda *a, **kw: fake_store
+    )
+    return fake_store
+
+
+def _seed_progress_row(
+    pg_conn, source_id: str, total: int, last_index: int,
+    attempt_id: str | None = None,
+) -> None:
+    pg_conn.execute(
+        text(
+            """
+            INSERT INTO ingest_chunk_progress
+                (source_id, total_chunks, embedded_chunks, last_index,
+                 attempt_id, last_updated)
+            VALUES (CAST(:sid AS uuid), :total, :emb, :idx, :att, now())
+            """
+        ),
+        {
+            "sid": source_id,
+            "total": total,
+            "emb": last_index + 1,
+            "idx": last_index,
+            "att": attempt_id,
+        },
+    )
+
+
+def _make_docs(n: int) -> list:
+    return [_LCDoc(page_content=f"chunk-{i}", metadata={"i": i}) for i in range(n)]
+
+
+@pytest.mark.unit
+class TestEmbedCheckpoint:
+    def test_fresh_run_records_progress_to_completion(
+        self, pg_conn, patch_pipeline_db, faiss_settings, tmp_path
+    ):
+        source_id = str(uuid.uuid4())
+        docs = _make_docs(5)
+        captured_chunks: list[str] = []
+
+        def _fake_add(store, doc, sid):
+            captured_chunks.append(doc.page_content)
+
+        # The retry decorator wraps the real fn. Patch the module-level name
+        # so our loop calls the spy directly.
+        import application.parser.embedding_pipeline as ep_mod
+
+        original = ep_mod.add_text_to_store_with_retry
+        ep_mod.add_text_to_store_with_retry = _fake_add
+        try:
+            ep_mod.embed_and_store_documents(
+                docs, str(tmp_path), source_id, MagicMock()
+            )
+        finally:
+            ep_mod.add_text_to_store_with_retry = original
+
+        # Faiss seeds the store with docs[0]; the loop picks up at idx=1.
+        assert captured_chunks == ["chunk-1", "chunk-2", "chunk-3", "chunk-4"]
+
+        row = pg_conn.execute(
+            text(
+                "SELECT total_chunks, embedded_chunks, last_index "
+                "FROM ingest_chunk_progress WHERE source_id = CAST(:sid AS uuid)"
+            ),
+            {"sid": source_id},
+        ).fetchone()
+        assert row is not None
+        m = dict(row._mapping)
+        # Loop wrote the last record at idx=4 over the full 5-doc list.
+        assert m["total_chunks"] == 5
+        assert m["embedded_chunks"] == 5
+        assert m["last_index"] == 4
+
+    def test_resume_skips_already_embedded_chunks(
+        self, pg_conn, patch_pipeline_db, faiss_settings, tmp_path
+    ):
+        """Pre-seed progress at ``last_index=2`` and assert chunks 0..2 are not re-embedded."""
+        import application.parser.embedding_pipeline as ep_mod
+
+        source_id = str(uuid.uuid4())
+        docs = _make_docs(6)
+        # Seed: last_index=2 means chunks at original indices 0..2 are done;
+        # resume should pick up at idx=3 (chunks 3, 4, 5).
+        _seed_progress_row(pg_conn, source_id, total=6, last_index=2)
+
+        captured_chunks: list[str] = []
+
+        def _fake_add(store, doc, sid):
+            captured_chunks.append(doc.page_content)
+
+        original = ep_mod.add_text_to_store_with_retry
+        ep_mod.add_text_to_store_with_retry = _fake_add
+        try:
+            ep_mod.embed_and_store_documents(
+                docs, str(tmp_path), source_id, MagicMock()
+            )
+        finally:
+            ep_mod.add_text_to_store_with_retry = original
+
+        # On resume the FAISS store is loaded from storage (no docs_init);
+        # the loop iterates the un-popped docs list starting at resume_index.
+        assert captured_chunks == ["chunk-3", "chunk-4", "chunk-5"]
+
+        row = pg_conn.execute(
+            text(
+                "SELECT embedded_chunks, last_index "
+                "FROM ingest_chunk_progress WHERE source_id = CAST(:sid AS uuid)"
+            ),
+            {"sid": source_id},
+        ).fetchone()
+        m = dict(row._mapping)
+        assert m["last_index"] == 5
+        assert m["embedded_chunks"] == 6
+
+    def test_faiss_resume_loads_existing_index_without_docs_init(
+        self, pg_conn, patch_pipeline_db, monkeypatch, tmp_path
+    ):
+        """Resuming a FAISS run must NOT pass ``docs_init`` (which would
+        overwrite the previously-saved index with a partial reset).
+        """
+        import application.parser.embedding_pipeline as ep_mod
+
+        monkeypatch.setattr(ep_mod.settings, "VECTOR_STORE", "faiss", raising=False)
+
+        captured_kwargs: list[dict] = []
+
+        def _capture_create(*args, **kwargs):
+            captured_kwargs.append(kwargs)
+            return MagicMock(name="vector_store")
+
+        monkeypatch.setattr(
+            ep_mod.VectorCreator, "create_vectorstore", _capture_create,
+        )
+
+        source_id = str(uuid.uuid4())
+        docs = _make_docs(6)
+        _seed_progress_row(pg_conn, source_id, total=6, last_index=2)
+
+        original = ep_mod.add_text_to_store_with_retry
+        ep_mod.add_text_to_store_with_retry = lambda store, doc, sid: None
+        try:
+            ep_mod.embed_and_store_documents(
+                docs, str(tmp_path), source_id, MagicMock()
+            )
+        finally:
+            ep_mod.add_text_to_store_with_retry = original
+
+        assert len(captured_kwargs) == 1
+        # No ``docs_init`` on resume — this is what triggers FaissStore to
+        # load the existing index from storage rather than minting a new
+        # single-doc store.
+        assert "docs_init" not in captured_kwargs[0]
+
+    def test_resume_keeps_existing_index_for_non_faiss(
+        self, pg_conn, patch_pipeline_db, monkeypatch, tmp_path
+    ):
+        """Non-faiss stores must NOT have ``delete_index`` called on resume."""
+        import application.parser.embedding_pipeline as ep_mod
+
+        monkeypatch.setattr(ep_mod.settings, "VECTOR_STORE", "qdrant", raising=False)
+
+        fake_store = MagicMock(name="vector_store")
+        monkeypatch.setattr(
+            ep_mod.VectorCreator, "create_vectorstore", lambda *a, **kw: fake_store
+        )
+
+        source_id = str(uuid.uuid4())
+        docs = _make_docs(4)
+        _seed_progress_row(pg_conn, source_id, total=4, last_index=1)
+
+        original = ep_mod.add_text_to_store_with_retry
+        ep_mod.add_text_to_store_with_retry = lambda store, doc, sid: None
+        try:
+            ep_mod.embed_and_store_documents(
+                docs, str(tmp_path), source_id, MagicMock()
+            )
+        finally:
+            ep_mod.add_text_to_store_with_retry = original
+
+        fake_store.delete_index.assert_not_called()
+
+    def test_same_attempt_id_resumes_from_checkpoint(
+        self, pg_conn, patch_pipeline_db, faiss_settings, tmp_path
+    ):
+        """A Celery autoretry passes the same ``self.request.id`` and
+        must resume from the persisted ``last_index``.
+        """
+        import application.parser.embedding_pipeline as ep_mod
+
+        source_id = str(uuid.uuid4())
+        docs = _make_docs(6)
+        _seed_progress_row(
+            pg_conn, source_id, total=6, last_index=2, attempt_id="att-A",
+        )
+
+        captured: list[str] = []
+        original = ep_mod.add_text_to_store_with_retry
+        ep_mod.add_text_to_store_with_retry = (
+            lambda store, doc, sid: captured.append(doc.page_content)
+        )
+        try:
+            ep_mod.embed_and_store_documents(
+                docs, str(tmp_path), source_id, MagicMock(),
+                attempt_id="att-A",
+            )
+        finally:
+            ep_mod.add_text_to_store_with_retry = original
+
+        # Same attempt → resume past the last persisted index.
+        assert captured == ["chunk-3", "chunk-4", "chunk-5"]
+
+    def test_different_attempt_id_resets_checkpoint(
+        self, pg_conn, patch_pipeline_db, faiss_settings, tmp_path
+    ):
+        """A fresh sync/reingest passes a new ``attempt_id`` and must
+        reset the checkpoint so the index is rebuilt from chunk 0.
+        """
+        import application.parser.embedding_pipeline as ep_mod
+
+        source_id = str(uuid.uuid4())
+        # Prior run completed: last_index=4 (chunks 0..4 embedded over a
+        # 5-doc list). New run brings 5 fresh docs under a different
+        # attempt_id.
+        _seed_progress_row(
+            pg_conn, source_id, total=5, last_index=4, attempt_id="att-old",
+        )
+        docs = _make_docs(5)
+
+        captured: list[str] = []
+        original = ep_mod.add_text_to_store_with_retry
+        ep_mod.add_text_to_store_with_retry = (
+            lambda store, doc, sid: captured.append(doc.page_content)
+        )
+        try:
+            ep_mod.embed_and_store_documents(
+                docs, str(tmp_path), source_id, MagicMock(),
+                attempt_id="att-new",
+            )
+        finally:
+            ep_mod.add_text_to_store_with_retry = original
+
+        # Fresh attempt → reset to chunk 0; FAISS branch seeds with
+        # docs[0] and the loop picks up at idx=1.
+        assert captured == ["chunk-1", "chunk-2", "chunk-3", "chunk-4"]
+
+        # Post-run state belongs to the new attempt.
+        row = pg_conn.execute(
+            text(
+                "SELECT total_chunks, embedded_chunks, last_index, attempt_id "
+                "FROM ingest_chunk_progress WHERE source_id = CAST(:sid AS uuid)"
+            ),
+            {"sid": source_id},
+        ).fetchone()
+        m = dict(row._mapping)
+        assert m["embedded_chunks"] == 5
+        assert m["last_index"] == 4
+        assert m["attempt_id"] == "att-new"
+
+    def test_completed_checkpoint_does_not_block_fresh_attempt(
+        self, pg_conn, patch_pipeline_db, faiss_settings, tmp_path
+    ):
+        """Regression: a completed-and-cached checkpoint from an earlier
+        upload must not silently no-op a later sync. Pre-fix the embed
+        loop saw ``loop_start >= total_docs`` and embedded zero chunks,
+        leaving stale vectors in place.
+        """
+        import application.parser.embedding_pipeline as ep_mod
+
+        source_id = str(uuid.uuid4())
+        # Upload finished cleanly with 5 chunks; checkpoint reflects done.
+        _seed_progress_row(
+            pg_conn, source_id, total=5, last_index=4, attempt_id="upload-1",
+        )
+        # Sync brings the same number of docs (the dangerous case where
+        # the old code's ``loop_start >= total_docs`` branch fired).
+        docs = _make_docs(5)
+
+        captured: list[str] = []
+        original = ep_mod.add_text_to_store_with_retry
+        ep_mod.add_text_to_store_with_retry = (
+            lambda store, doc, sid: captured.append(doc.page_content)
+        )
+        try:
+            ep_mod.embed_and_store_documents(
+                docs, str(tmp_path), source_id, MagicMock(),
+                attempt_id="sync-2",
+            )
+        finally:
+            ep_mod.add_text_to_store_with_retry = original
+
+        # All non-seed chunks re-embedded under the new attempt.
+        assert captured == ["chunk-1", "chunk-2", "chunk-3", "chunk-4"]
+
+    def test_legacy_null_attempt_id_resumes_against_null_caller(
+        self, pg_conn, patch_pipeline_db, faiss_settings, tmp_path
+    ):
+        """Pre-migration rows have ``attempt_id=NULL``; legacy callers
+        (or tests) that pass no ``attempt_id`` must still resume against
+        them — IS NOT DISTINCT FROM treats NULL/NULL as equal.
+        """
+        import application.parser.embedding_pipeline as ep_mod
+
+        source_id = str(uuid.uuid4())
+        _seed_progress_row(
+            pg_conn, source_id, total=4, last_index=1, attempt_id=None,
+        )
+        docs = _make_docs(4)
+
+        captured: list[str] = []
+        original = ep_mod.add_text_to_store_with_retry
+        ep_mod.add_text_to_store_with_retry = (
+            lambda store, doc, sid: captured.append(doc.page_content)
+        )
+        try:
+            ep_mod.embed_and_store_documents(
+                docs, str(tmp_path), source_id, MagicMock(),
+            )
+        finally:
+            ep_mod.add_text_to_store_with_retry = original
+
+        # Resumed past last_index=1.
+        assert captured == ["chunk-2", "chunk-3"]
+
+    def test_single_chunk_faiss_records_seeded_doc(
+        self, pg_conn, patch_pipeline_db, faiss_settings, tmp_path
+    ):
+        """Regression: a 1-doc FAISS ingest seeds with ``docs[0]`` and
+        the loop runs zero iterations. Pre-fix, no ``_record_progress``
+        call ever ran, ``embedded_chunks`` stayed at 0, and
+        ``assert_index_complete`` raised on every retry until the
+        poison-loop guard finalised the row. Post-fix, the seed is
+        recorded immediately so ``embedded == total == 1``.
+        """
+        import application.parser.embedding_pipeline as ep_mod
+
+        source_id = str(uuid.uuid4())
+        docs = _make_docs(1)
+
+        original = ep_mod.add_text_to_store_with_retry
+        ep_mod.add_text_to_store_with_retry = lambda store, doc, sid: None
+        try:
+            ep_mod.embed_and_store_documents(
+                docs, str(tmp_path), source_id, MagicMock(),
+                attempt_id="att-single",
+            )
+        finally:
+            ep_mod.add_text_to_store_with_retry = original
+
+        row = pg_conn.execute(
+            text(
+                "SELECT total_chunks, embedded_chunks, last_index, attempt_id "
+                "FROM ingest_chunk_progress WHERE source_id = CAST(:sid AS uuid)"
+            ),
+            {"sid": source_id},
+        ).fetchone()
+        assert row is not None
+        m = dict(row._mapping)
+        assert m["total_chunks"] == 1
+        assert m["embedded_chunks"] == 1
+        assert m["last_index"] == 0
+        assert m["attempt_id"] == "att-single"
+
+        # Tripwire passes — assert_index_complete reads the same row.
+        ep_mod.assert_index_complete(source_id)
+
+    def test_multi_chunk_faiss_seed_record_is_overwritten_correctly(
+        self, pg_conn, patch_pipeline_db, faiss_settings, tmp_path
+    ):
+        """The new seed-record call must not break multi-chunk runs:
+        the loop's per-iteration record overshoots correctly (counts
+        seed + iterations) and the final state is ``embedded=total``.
+        """
+        import application.parser.embedding_pipeline as ep_mod
+
+        source_id = str(uuid.uuid4())
+        docs = _make_docs(4)
+
+        original = ep_mod.add_text_to_store_with_retry
+        ep_mod.add_text_to_store_with_retry = lambda store, doc, sid: None
+        try:
+            ep_mod.embed_and_store_documents(
+                docs, str(tmp_path), source_id, MagicMock(),
+                attempt_id="att-multi",
+            )
+        finally:
+            ep_mod.add_text_to_store_with_retry = original
+
+        row = pg_conn.execute(
+            text(
+                "SELECT total_chunks, embedded_chunks, last_index "
+                "FROM ingest_chunk_progress WHERE source_id = CAST(:sid AS uuid)"
+            ),
+            {"sid": source_id},
+        ).fetchone()
+        m = dict(row._mapping)
+        assert m["total_chunks"] == 4
+        assert m["embedded_chunks"] == 4
+        assert m["last_index"] == 3
+
+
+@pytest.mark.unit
+class TestIngestHeartbeat:
+    def test_loop_bumps_last_updated_then_exits(
+        self, pg_conn, patch_worker_db, monkeypatch
+    ):
+        """One tick of the heartbeat must move ``last_updated`` forward."""
+        from application import worker
+
+        source_id = str(uuid.uuid4())
+        # Seed a row with ``last_updated`` deliberately in the past so we
+        # can assert the heartbeat tick moves it forward.
+        pg_conn.execute(
+            text(
+                """
+                INSERT INTO ingest_chunk_progress
+                    (source_id, total_chunks, embedded_chunks, last_index, last_updated)
+                VALUES (CAST(:sid AS uuid), 1, 0, -1, now() - interval '1 hour')
+                """
+            ),
+            {"sid": source_id},
+        )
+        before = pg_conn.execute(
+            text(
+                "SELECT last_updated FROM ingest_chunk_progress "
+                "WHERE source_id = CAST(:sid AS uuid)"
+            ),
+            {"sid": source_id},
+        ).fetchone()._mapping["last_updated"]
+
+        stop_event = threading.Event()
+        # First call returns False -> loop body runs once. Second returns
+        # True -> loop exits cleanly.
+        wait_returns = iter([False, True])
+        monkeypatch.setattr(stop_event, "wait", lambda interval: next(wait_returns))
+
+        worker._ingest_heartbeat_loop(source_id, stop_event, interval=0)
+
+        after = pg_conn.execute(
+            text(
+                "SELECT last_updated FROM ingest_chunk_progress "
+                "WHERE source_id = CAST(:sid AS uuid)"
+            ),
+            {"sid": source_id},
+        ).fetchone()._mapping["last_updated"]
+        assert after > before
+
+    def test_loop_swallows_db_errors(self, monkeypatch):
+        """A failing DB call must not crash the daemon — it should keep ticking."""
+        from application import worker
+
+        @contextmanager
+        def _broken_session():
+            raise RuntimeError("boom")
+            yield  # pragma: no cover
+
+        monkeypatch.setattr(worker, "db_session", _broken_session)
+
+        stop_event = threading.Event()
+        wait_returns = iter([False, False, True])
+        monkeypatch.setattr(stop_event, "wait", lambda interval: next(wait_returns))
+
+        # Should not raise — failures are logged, loop continues.
+        worker._ingest_heartbeat_loop("00000000-0000-0000-0000-000000000000", stop_event, interval=0)
+
+    def test_start_and_stop_helpers_join_quickly(self, monkeypatch):
+        """``_start_ingest_heartbeat`` + ``_stop_ingest_heartbeat`` must not hang."""
+        from application import worker
+
+        @contextmanager
+        def _noop_session():
+            yield MagicMock()
+
+        monkeypatch.setattr(worker, "db_session", _noop_session)
+
+        thread, stop_event = worker._start_ingest_heartbeat(
+            "00000000-0000-0000-0000-000000000000"
+        )
+        assert thread.is_alive()
+        worker._stop_ingest_heartbeat(thread, stop_event)
+        assert not thread.is_alive()

tests/worker/test_ingest_connector.py

@@ -9,6 +9,7 @@ is the sync variant.
 
 from __future__ import annotations
 
+import uuid
 from unittest.mock import MagicMock
 
 import pytest
@@ -57,7 +58,7 @@ def _mock_connector_pipeline(monkeypatch):
     monkeypatch.setattr(
         worker,
         "embed_and_store_documents",
-        lambda docs, full_path, source_id, task: None,
+        lambda docs, full_path, source_id, task, **kw: None,
     )
     monkeypatch.setattr(
         worker, "upload_index", lambda full_path, file_data: None
@@ -104,6 +105,74 @@ class TestIngestConnectorSyncUpdatesDate:
 
         refreshed = SourcesRepository(pg_conn).get(source_id, "dave")
         assert refreshed is not None
-        assert refreshed["date"] > old_date, (
+        # row_to_dict coerces datetimes to ISO 8601 strings; UTC
+        # timezone-aware ISO strings sort chronologically.
+        assert refreshed["date"] > old_date.isoformat(), (
             "ingest_connector(sync) should push sources.date forward"
         )
+
+
+@pytest.mark.unit
+class TestIngestConnectorDeterministicSourceId:
+    """Upload-mode ``ingest_connector`` derives a stable ``source_id`` from the key."""
+
+    def test_uses_uuid5_when_idempotency_key_present(
+        self,
+        task_self,
+        monkeypatch,
+        _mock_connector_pipeline,
+    ):
+        from application import worker
+
+        captured: list[dict] = []
+        monkeypatch.setattr(
+            worker, "upload_index",
+            lambda full_path, file_data: captured.append(file_data),
+        )
+
+        for _ in range(2):
+            worker.ingest_connector(
+                task_self,
+                "gdrive-folder",
+                "dave",
+                "google_drive",
+                session_token="tok",
+                file_ids=["f1"],
+                folder_ids=[],
+                operation_mode="upload",
+                idempotency_key="abc",
+            )
+
+        expected = str(uuid.uuid5(worker.DOCSGPT_INGEST_NAMESPACE, "abc"))
+        assert len(captured) == 2
+        assert captured[0]["id"] == expected
+        assert captured[1]["id"] == expected
+
+    def test_falls_back_to_uuid4_without_key(
+        self,
+        task_self,
+        monkeypatch,
+        _mock_connector_pipeline,
+    ):
+        from application import worker
+
+        captured: list[dict] = []
+        monkeypatch.setattr(
+            worker, "upload_index",
+            lambda full_path, file_data: captured.append(file_data),
+        )
+
+        for _ in range(2):
+            worker.ingest_connector(
+                task_self,
+                "gdrive-folder",
+                "dave",
+                "google_drive",
+                session_token="tok",
+                file_ids=["f1"],
+                folder_ids=[],
+                operation_mode="upload",
+            )
+
+        assert len(captured) == 2
+        assert captured[0]["id"] != captured[1]["id"]

tests/worker/test_ingest_worker.py

@@ -16,6 +16,8 @@ backend-facing write depends on.
 
 from __future__ import annotations
 
+import uuid
+from io import BytesIO
 from unittest.mock import MagicMock
 
 import pytest
@@ -23,6 +25,51 @@ import pytest
 from application.parser.schema.base import Document
 
 
+def _patch_ingest_pipeline(monkeypatch, captured):
+    """Stub the non-PG boundaries used by ``ingest_worker``.
+
+    Storage, the directory reader, the embedding pipeline, and
+    ``upload_index`` are all replaced with cheap fakes so the worker
+    runs end-to-end inside the test process. ``captured`` is appended
+    with each ``upload_index`` payload so callers can assert on the
+    derived ``source_id``.
+    """
+    from application import worker
+
+    fake_storage = MagicMock(name="storage")
+    fake_storage.is_directory.return_value = False
+    fake_storage.get_file.return_value = BytesIO(b"hello")
+    monkeypatch.setattr(
+        worker.StorageCreator, "get_storage", lambda: fake_storage
+    )
+
+    fake_reader = MagicMock(name="reader")
+    fake_reader.load_data.return_value = [
+        Document(text="hello body", extra_info={"source": "a.txt"})
+    ]
+    fake_reader.directory_structure = {
+        "a.txt": {
+            "type": "text/plain",
+            "size_bytes": 5,
+            "token_count": 2,
+        }
+    }
+    monkeypatch.setattr(
+        worker, "SimpleDirectoryReader", lambda *a, **kw: fake_reader
+    )
+
+    monkeypatch.setattr(
+        worker,
+        "embed_and_store_documents",
+        lambda docs, full_path, source_id, task, **kw: None,
+    )
+
+    monkeypatch.setattr(
+        worker, "upload_index",
+        lambda full_path, file_data: captured.append(file_data),
+    )
+
+
 @pytest.mark.unit
 class TestIngestWorker:
     def test_invokes_upload_index_with_expected_payload(
@@ -30,43 +77,8 @@ class TestIngestWorker:
     ):
         from application import worker
 
-        # Storage: pretend a single file was uploaded at ``inputs/.../a.txt``.
-        from io import BytesIO
-        fake_storage = MagicMock(name="storage")
-        fake_storage.is_directory.return_value = False
-        fake_storage.get_file.return_value = BytesIO(b"hello")
-        monkeypatch.setattr(
-            worker.StorageCreator, "get_storage", lambda: fake_storage
-        )
-
-        # Reader: return a parsed doc.
-        fake_reader = MagicMock(name="reader")
-        fake_reader.load_data.return_value = [
-            Document(text="hello body", extra_info={"source": "a.txt"})
-        ]
-        fake_reader.directory_structure = {
-            "a.txt": {
-                "type": "text/plain",
-                "size_bytes": 5,
-                "token_count": 2,
-            }
-        }
-        monkeypatch.setattr(
-            worker, "SimpleDirectoryReader", lambda *a, **kw: fake_reader
-        )
-
-        monkeypatch.setattr(
-            worker,
-            "embed_and_store_documents",
-            lambda docs, full_path, source_id, task: None,
-        )
-
         captured: list[dict] = []
-
-        def _fake_upload(full_path, file_data):
-            captured.append(file_data)
-
-        monkeypatch.setattr(worker, "upload_index", _fake_upload)
+        _patch_ingest_pipeline(monkeypatch, captured)
 
         result = worker.ingest_worker(
             task_self,
@@ -91,3 +103,126 @@ class TestIngestWorker:
         assert payload["type"] == "local"
         # A fresh source UUID is minted for the backend /upload_index route.
         assert payload["id"]
+
+
+@pytest.mark.unit
+class TestIngestWorkerDeterministicSourceId:
+    """Retried ingests with the same key should land on the same source row."""
+
+    def test_uses_uuid5_when_idempotency_key_present(
+        self, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        captured: list[dict] = []
+        _patch_ingest_pipeline(monkeypatch, captured)
+
+        worker.ingest_worker(
+            task_self,
+            directory="inputs",
+            formats=[".txt"],
+            job_name="job1",
+            file_path="inputs/eve/job1/a.txt",
+            filename="a.txt",
+            user="eve",
+            idempotency_key="abc",
+        )
+        worker.ingest_worker(
+            task_self,
+            directory="inputs",
+            formats=[".txt"],
+            job_name="job1",
+            file_path="inputs/eve/job1/a.txt",
+            filename="a.txt",
+            user="eve",
+            idempotency_key="abc",
+        )
+
+        expected = str(
+            uuid.uuid5(worker.DOCSGPT_INGEST_NAMESPACE, "abc")
+        )
+        assert len(captured) == 2
+        assert captured[0]["id"] == expected
+        assert captured[1]["id"] == expected
+
+    def test_falls_back_to_uuid4_without_key(
+        self, patch_worker_db, task_self, monkeypatch
+    ):
+        from application import worker
+
+        captured: list[dict] = []
+        _patch_ingest_pipeline(monkeypatch, captured)
+
+        for _ in range(2):
+            worker.ingest_worker(
+                task_self,
+                directory="inputs",
+                formats=[".txt"],
+                job_name="job1",
+                file_path="inputs/eve/job1/a.txt",
+                filename="a.txt",
+                user="eve",
+            )
+
+        assert len(captured) == 2
+        # Random uuid4 fallback: two runs must produce different ids.
+        assert captured[0]["id"] != captured[1]["id"]
+
+    def test_double_ingest_writes_one_source_row(
+        self, pg_conn, patch_worker_db, task_self, monkeypatch
+    ):
+        """Run the worker body twice with the same key; assert the
+        backend-side ``upload_index`` would persist one row, not two.
+
+        ``upload_index`` is the actual backend write path; the worker
+        feeds it a ``file_data`` payload keyed by the derived id. We
+        emulate that path locally (``SourcesRepository.create``) on the
+        first call and assert the second call hits the existing-row
+        branch instead of inserting again.
+        """
+        from application import worker
+        from application.storage.db.repositories.sources import SourcesRepository
+
+        captured: list[dict] = []
+        _patch_ingest_pipeline(monkeypatch, captured)
+
+        def _persist(full_path, file_data):
+            captured.append(file_data)
+            repo = SourcesRepository(pg_conn)
+            existing = repo.get(file_data["id"], file_data["user"])
+            if existing is None:
+                repo.create(
+                    file_data["name"],
+                    source_id=file_data["id"],
+                    user_id=file_data["user"],
+                    type=file_data["type"],
+                    retriever=file_data["retriever"],
+                    tokens=file_data["tokens"],
+                )
+
+        monkeypatch.setattr(worker, "upload_index", _persist)
+
+        for _ in range(2):
+            worker.ingest_worker(
+                task_self,
+                directory="inputs",
+                formats=[".txt"],
+                job_name="job1",
+                file_path="inputs/eve/job1/a.txt",
+                filename="a.txt",
+                user="eve",
+                idempotency_key="dedupe-key",
+            )
+
+        expected = str(
+            uuid.uuid5(worker.DOCSGPT_INGEST_NAMESPACE, "dedupe-key")
+        )
+        assert captured[0]["id"] == expected
+        assert captured[1]["id"] == expected
+
+        # Exactly one row in PG for that derived id.
+        result = pg_conn.exec_driver_sql(
+            "SELECT count(*) FROM sources WHERE id = %s",
+            (expected,),
+        ).fetchone()
+        assert result[0] == 1

tests/worker/test_remote_worker.py

@@ -14,6 +14,7 @@ assert one seeded row is discovered and forwarded.
 from __future__ import annotations
 
 import os
+import uuid
 from unittest.mock import MagicMock
 
 import pytest
@@ -41,7 +42,7 @@ def _mock_remote_pipeline(monkeypatch):
     monkeypatch.setattr(
         worker,
         "embed_and_store_documents",
-        lambda docs, full_path, source_id, task: None,
+        lambda docs, full_path, source_id, task, **kw: None,
     )
     monkeypatch.setattr(
         worker, "upload_index", lambda full_path, file_data: None
@@ -90,7 +91,9 @@ class TestRemoteWorkerSyncUpdatesDate:
 
         refreshed = SourcesRepository(pg_conn).get(source_id, "bob")
         assert refreshed is not None
-        assert refreshed["date"] > old_date, (
+        # row_to_dict coerces datetimes to ISO 8601 strings; UTC
+        # timezone-aware ISO strings sort chronologically.
+        assert refreshed["date"] > old_date.isoformat(), (
             "remote_worker(sync) should push sources.date forward"
         )
 
@@ -210,3 +213,69 @@ class TestRemoteWorkerPathTraversal:
             f"rmtree target {rmtree_targets[0]} escaped {user_root}"
         )
         assert malicious_name not in "".join(created_paths + deleted_paths)
+
+
+@pytest.mark.unit
+class TestRemoteWorkerDeterministicSourceId:
+    """Upload-mode ``remote_worker`` derives a stable ``source_id`` from the key."""
+
+    def test_uses_uuid5_when_idempotency_key_present(
+        self,
+        tmp_path,
+        task_self,
+        monkeypatch,
+        _mock_remote_pipeline,
+    ):
+        from application import worker
+
+        captured: list[dict] = []
+        monkeypatch.setattr(
+            worker, "upload_index",
+            lambda full_path, file_data: captured.append(file_data),
+        )
+
+        for _ in range(2):
+            worker.remote_worker(
+                task_self,
+                {"urls": ["http://example.com"]},
+                "feed",
+                "bob",
+                "crawler",
+                directory=str(tmp_path / "temp"),
+                operation_mode="upload",
+                idempotency_key="abc",
+            )
+
+        expected = str(uuid.uuid5(worker.DOCSGPT_INGEST_NAMESPACE, "abc"))
+        assert len(captured) == 2
+        assert captured[0]["id"] == expected
+        assert captured[1]["id"] == expected
+
+    def test_falls_back_to_uuid4_without_key(
+        self,
+        tmp_path,
+        task_self,
+        monkeypatch,
+        _mock_remote_pipeline,
+    ):
+        from application import worker
+
+        captured: list[dict] = []
+        monkeypatch.setattr(
+            worker, "upload_index",
+            lambda full_path, file_data: captured.append(file_data),
+        )
+
+        for _ in range(2):
+            worker.remote_worker(
+                task_self,
+                {"urls": ["http://example.com"]},
+                "feed",
+                "bob",
+                "crawler",
+                directory=str(tmp_path / "temp"),
+                operation_mode="upload",
+            )
+
+        assert len(captured) == 2
+        assert captured[0]["id"] != captured[1]["id"]