fix: little more tool call hardening

This commit is contained in:
Alex committed 2026-08-21 15:01:45 +01:00
1 parent 4a4f485781
commit 114585cd7d
14 files changed
+961 -22

No files matched your search

+76 -3
View File
@@ -443,7 +443,10 @@ class TestStuckProposedToolCalls:
{"id": "cp-1"},
).fetchone()
assert row[0] == "failed"
assert "side effect status unknown" in (row[1] or "")
# The row is written immediately before the tool runs, so an abandoned
# stream leaves no side effect at all — the message must not imply one.
assert "never advanced past 'proposed'" in (row[1] or "")
assert "no side effect was committed" in (row[1] or "")
assert any(
getattr(rec, "alert", None) == "reconciler_tool_call_failed_proposed"
@@ -498,8 +501,11 @@ class TestStuckExecutedToolCalls:
{"id": "ce-1"},
).fetchone()
assert row[0] == "failed"
assert "executed-not-confirmed" in (row[1] or "")
assert "manual cleanup" in (row[1] or "")
# The tool SUCCEEDED here; confirmation only lands when the whole turn
# finalizes, so a healthy long turn reaches this state and the message
# must not assert that cleanup is required.
assert "executed but not confirmed" in (row[1] or "")
assert "The tool SUCCEEDED" in (row[1] or "")
assert any(
getattr(rec, "alert", None) == "reconciler_tool_call_failed_executed"
for rec in caplog.records
@@ -950,3 +956,70 @@ class TestPostgresUriMissing:
"tool_calls_failed": 0,
"skipped": "POSTGRES_URI not set",
}
# ---------------------------------------------------------------------------
# Publish-after-commit durability — a sweep that raises must not swallow the
# events whose DB writes already landed.
# ---------------------------------------------------------------------------
class TestEventsSurviveAFailedSweep:
"""Regression: 2026-08-21, an Azure DNS blip took Postgres unreachable
mid-tick. ``run_reconciliation`` commits each sweep in its own transaction
but fans out queued user-facing events once, at the end — so an exception
in a later sweep used to discard events for rows that were already
terminal. The loss is permanent, because each sweep filters out the rows it
just escalated, so no later tick re-finds them.
"""
def test_queued_events_are_published_when_a_later_sweep_raises(self, pg_conn):
from application.api.user.reconciliation import run_reconciliation
# A stalled ingest: escalated (and its event queued) by the sweep that
# runs before the one we blow up.
source_id = "1a000000-0000-0000-0000-0000000000e1"
_seed_source(pg_conn, source_id=source_id, user_id="u-blip")
_seed_ingest_progress(pg_conn, source_id=source_id, embedded=2, total=50)
published: list = []
def _capture(user_id, event_type, payload, *, scope=None):
published.append((user_id, event_type, payload, scope))
return "1-0"
calls = {"n": 0}
@contextmanager
def _fake_begin():
calls["n"] += 1
# Fail a sweep AFTER the ingest sweep has committed its row.
if calls["n"] == 6:
raise RuntimeError("connection is bad: Network is unreachable")
yield pg_conn
fake_engine = MagicMock()
fake_engine.begin = _fake_begin
with patch(
"application.api.user.reconciliation.get_engine",
return_value=fake_engine,
), patch(
"application.events.publisher.publish_user_event", _capture
):
with pytest.raises(RuntimeError):
run_reconciliation()
# The DB write landed...
status = pg_conn.execute(
text(
"SELECT status FROM ingest_chunk_progress "
"WHERE source_id = CAST(:sid AS uuid)"
),
{"sid": source_id},
).fetchone()[0]
assert status == "stalled"
# ...and so did its event. Without the ``finally`` the user's upload
# toast would spin on "Training..." forever.
assert [p[1] for p in published] == ["source.ingest.failed"]