fix: more stability for durable tasks, retry strategy, refactor dead

code

tests/storage/db/repositories/test_pending_tool_state.py

@@ -222,3 +222,56 @@ class TestRevertStaleResuming:
         loaded = repo.load_state(conv["id"], "user-1")
         assert loaded is not None
         assert loaded["status"] == "pending"
+
+
+class TestReleaseClaim:
+    def test_returns_a_claimed_row_to_pending(self, pg_conn):
+        """A failed resume must hand the claim back immediately.
+
+        Without this the only way back is ``revert_stale_resuming``'s 600 s
+        grace, and ``load_state`` ignores ``resuming`` rows — so the paused
+        turn stays resumable but invisible and the user is locked out of their
+        own conversation for ten minutes.
+        """
+        conv = _conv(pg_conn)
+        repo = _repo(pg_conn)
+        repo.save_state(conv["id"], "user-1", **_sample_state())
+        assert repo.claim_state(conv["id"], "user-1") is not None
+        assert repo.load_state(conv["id"], "user-1") is None
+
+        assert repo.release_claim(conv["id"], "user-1") is True
+
+        reclaimed = repo.load_state(conv["id"], "user-1")
+        assert reclaimed is not None
+        assert reclaimed["status"] == "pending"
+        assert reclaimed["resumed_at"] is None
+        # ...and the next resume attempt can take it straight away.
+        assert repo.claim_state(conv["id"], "user-1") is not None
+
+    def test_noop_on_a_pending_row(self, pg_conn):
+        """Only a claim can be released; a live pending row is left alone."""
+        conv = _conv(pg_conn)
+        repo = _repo(pg_conn)
+        repo.save_state(conv["id"], "user-1", **_sample_state())
+        assert repo.release_claim(conv["id"], "user-1") is False
+        assert repo.load_state(conv["id"], "user-1")["status"] == "pending"
+
+    def test_noop_when_the_row_is_gone(self, pg_conn):
+        """Must not resurrect a row another request deleted."""
+        conv = _conv(pg_conn)
+        repo = _repo(pg_conn)
+        repo.save_state(conv["id"], "user-1", **_sample_state())
+        repo.claim_state(conv["id"], "user-1")
+        repo.delete_state(conv["id"], "user-1")
+
+        assert repo.release_claim(conv["id"], "user-1") is False
+        assert repo.load_state_any(conv["id"], "user-1") is None
+
+    def test_scoped_to_the_owning_user(self, pg_conn):
+        conv = _conv(pg_conn)
+        repo = _repo(pg_conn)
+        repo.save_state(conv["id"], "user-1", **_sample_state())
+        repo.claim_state(conv["id"], "user-1")
+
+        assert repo.release_claim(conv["id"], "someone-else") is False
+        assert repo.load_state_any(conv["id"], "user-1")["status"] == "resuming"

tests/storage/db/repositories/test_truncate_supersede.py

@@ -118,3 +118,86 @@ class TestTruncateAfterSupersede:
     def test_invalid_conversation_id_is_rejected(self, pg_conn):
         repo = ConversationsRepository(pg_conn)
         assert repo.truncate_after("not-a-uuid", keep_up_to=0) == 0
+
+
+class TestSupersedeTombstone:
+    """``NOT_FOUND`` alone cannot tell a replaced turn from a lost answer.
+
+    The cancel flag is a ``threading.Event`` in the superseding request's own
+    process, so a stream in another worker never sees it and runs to
+    completion. Its late finalize then hits a row that is gone — routine on the
+    retry/edit path, and previously logged as an ERROR indistinguishable from a
+    genuinely orphaned answer.
+    """
+
+    def test_truncate_tombstones_every_row_it_deletes(self, pg_conn):
+        repo = ConversationsRepository(pg_conn)
+        conv = repo.create("u", "tombstone")
+        live = _seed(pg_conn, conv["id"], 0, "streaming")
+        done = _seed(pg_conn, conv["id"], 1, "complete")
+
+        repo.truncate_after(conv["id"], keep_up_to=-1)
+
+        # Both, not just the in-flight one: the UPDATE's status filter is
+        # narrower than the DELETE's, so the stamp alone would miss `done`.
+        assert repo.was_superseded(live) is True
+        assert repo.was_superseded(done) is True
+
+    def test_tombstone_survives_the_delete_that_created_it(self, pg_conn):
+        """No FK to conversation_messages, or the CASCADE would take it."""
+        repo = ConversationsRepository(pg_conn)
+        conv = repo.create("u", "cascade")
+        live = _seed(pg_conn, conv["id"], 0, "streaming")
+
+        repo.truncate_after(conv["id"], keep_up_to=-1)
+
+        assert _row(pg_conn, live) is None
+        assert repo.was_superseded(live) is True
+
+    def test_an_untouched_message_has_no_tombstone(self, pg_conn):
+        """A genuinely orphaned answer must still reach the ERROR path."""
+        repo = ConversationsRepository(pg_conn)
+        conv = repo.create("u", "kept")
+        kept = _seed(pg_conn, conv["id"], 0, "streaming")
+        dropped = _seed(pg_conn, conv["id"], 1, "streaming")
+
+        repo.truncate_after(conv["id"], keep_up_to=0)
+
+        assert repo.was_superseded(kept) is False
+        assert repo.was_superseded(dropped) is True
+
+    def test_was_superseded_shrugs_off_a_non_uuid(self, pg_conn):
+        """Shape-gate: a legacy ObjectId must not poison the transaction."""
+        repo = ConversationsRepository(pg_conn)
+        assert repo.was_superseded("507f1f77bcf86cd799439011") is False
+        assert repo.was_superseded("") is False
+
+    def test_truncate_is_repeatable(self, pg_conn):
+        """A second truncate over the same range must not raise on the PK."""
+        repo = ConversationsRepository(pg_conn)
+        conv = repo.create("u", "twice")
+        first = _seed(pg_conn, conv["id"], 0, "streaming")
+        repo.truncate_after(conv["id"], keep_up_to=-1)
+        _seed(pg_conn, conv["id"], 0, "streaming")
+        repo.truncate_after(conv["id"], keep_up_to=-1)
+        assert repo.was_superseded(first) is True
+
+    def test_cleanup_sweeps_old_tombstones_only(self, pg_conn):
+        repo = ConversationsRepository(pg_conn)
+        conv = repo.create("u", "sweep")
+        recent = _seed(pg_conn, conv["id"], 0, "streaming")
+        repo.truncate_after(conv["id"], keep_up_to=-1)
+        stale = _seed(pg_conn, conv["id"], 0, "streaming")
+        repo.truncate_after(conv["id"], keep_up_to=-1)
+        pg_conn.execute(
+            text(
+                "UPDATE superseded_messages "
+                "SET superseded_at = clock_timestamp() - make_interval(days => 30) "
+                "WHERE message_id = CAST(:mid AS uuid)"
+            ),
+            {"mid": stale},
+        )
+
+        assert repo.cleanup_superseded_older_than(14) == 1
+        assert repo.was_superseded(stale) is False
+        assert repo.was_superseded(recent) is True

tests/storage/db/test_migration_0029.py

@@ -29,6 +29,15 @@ def _run_alembic(url: str, *args: str) -> None:
     )
 
 
+def _alembic_history(url: str) -> str:
+    return subprocess.check_output(
+        [sys.executable, "-m", "alembic", "-c", str(_alembic_ini()), "history"],
+        timeout=60,
+        env={**os.environ, "POSTGRES_URI": url},
+        text=True,
+    )
+
+
 def _alembic_heads(url: str) -> list[str]:
     out = subprocess.check_output(
         [sys.executable, "-m", "alembic", "-c", str(_alembic_ini()), "heads"],
@@ -61,10 +70,16 @@ def _table_exists(conn, table: str) -> bool:
 
 class TestMigration0029RoundTrip:
     def test_single_head(self, pg_engine):
+        """One head, and 0029 still on the path to it.
+
+        Deliberately does NOT assert 0029 *is* the head — every later
+        migration would break that — only that the chain stays linear and
+        0029 has not been orphaned off it.
+        """
         url = pg_engine.url.render_as_string(hide_password=False)
         heads = _alembic_heads(url)
         assert len(heads) == 1, f"expected one alembic head, got {heads}"
-        assert _0029 in heads[0]
+        assert _0029 in _alembic_history(url)
 
     def test_upgrade_creates_column_and_table(self, pg_engine):
         with pg_engine.connect() as conn: