Keep tool output out of span errors and fix trace status and listing

Tool results, denial comments and tool exception text now reach a span only
as a capture-gated preview; span.error is a fixed message, since it is
stored and exported regardless of content settings. A turn whose stream
yields an error event is recorded as failed. Search traces are listed by a
query copied into the small summary column, so the Logs timeline never
reads the spans JSONB. Adds GraphRAG span tests.
This commit is contained in:
arc53-machine committed 2026-09-23 18:11:19 +01:00
1 parent 285fdc914c
commit efec1a2f83
9 files changed
+134 -13

No files matched your search

+11
View File
@@ -132,6 +132,17 @@ class TestTraceLifecycle:
assert any('"type": "error"' in s for s in stream)
assert flushed[0].status == "error"
def test_yielded_error_marks_trace_error(self, flask_app, mock_mongo_db):
"""A failed workflow node yields an error event instead of raising."""
from docsgpt.api.answer.routes.base import BaseAnswerResource
with flask_app.app_context(), _captured_flushes() as flushed:
_run(
BaseAnswerResource(),
_agent([{"type": "error", "error": "node failed"}]),
)
assert flushed[0].status == "error"
def test_abandoned_stream_still_flushes(self, flask_app, mock_mongo_db):
from docsgpt.api.answer.routes.base import BaseAnswerResource
+8 -1
View File
@@ -167,7 +167,14 @@ class TestLogsTraceSummaries:
assert row["trace"]["ref"] == {"field": "activity_id", "value": "act-1"}
def test_search_traces_are_listed(self, app, pg_conn):
record = _trace(pg_conn, request_id=None, source="mcp", name="mcp", status="error")
record = _trace(
pg_conn,
request_id=None,
source="mcp",
name="mcp",
status="error",
summary={"llm_calls": 0, "query": "how do I deploy"},
)
_trace(pg_conn, request_id=None, source="graph_extraction", name="graph_extraction Docs")
search_rows = _logs(app, pg_conn, "owner", {"event_type": "search"}).json["logs"]
assert len(search_rows) == 1
+38
View File
@@ -1021,3 +1021,41 @@ class TestPersonalizedPageRankWithoutScipy:
docs = rag._get_data()
assert [doc["text"] for doc in docs] == ["near", "far"]
class TestTraceSpans:
"""GraphRAG searches and query embeddings are recorded in the execution trace."""
@pytest.fixture(autouse=True)
def _trace(self, monkeypatch):
from docsgpt import tracing
from docsgpt.core.settings import settings
monkeypatch.setattr(settings, "TRACES_ENABLED", True)
monkeypatch.setattr(settings, "TRACES_CAPTURE_CONTENT", True)
self.trace = tracing.start_trace(source="stream", capture_otel_context=False)
with tracing.activate(self.trace):
yield
def test_search_is_a_retrieval_span(self, _patch_llm_creator):
rag = _make_retriever()
docs = [{"text": "alpha", "title": "Doc A", "source": "a.md"}]
with patch.object(rag, "_get_data", return_value=docs):
assert rag.search("new question") == docs
(span,) = [s for s in self.trace.spans if s.kind == "retrieval"]
assert span.name == "retrieval GraphRAGRetriever"
assert span.attributes["docsgpt.retriever"] == "GraphRAGRetriever"
assert span.attributes["gen_ai.data_source.id"] == "src1"
assert span.attributes["docsgpt.chunk_count"] == 1
assert span.previews["chunks"][0]["title"] == "Doc A"
def test_embed_query_is_an_embedding_span(self):
embedder = Mock()
embedder.embed_query.return_value = [0.5]
with patch(
"docsgpt.retriever.graph_rag.get_embeddings", return_value=embedder
):
assert GraphRAGRetriever._embed_query(object(), "q") == [0.5]
(span,) = self.trace.spans
assert span.kind == "embedding"
assert span.attributes["gen_ai.operation.name"] == "embeddings"
+25 -3
View File
@@ -82,7 +82,9 @@ class TestExecute:
_drain(executor.execute({}, ToolCall(id="c", name="x", arguments="{}"), "L"))
span = trace.spans[0]
assert span.status == "error"
assert span.error == "no such tool"
# Tool output stays in the capture-gated preview, never in ``error``.
assert span.error == "Tool call failed (ToolError)"
assert span.previews["result"] == "no such tool"
def test_raised_error(self, trace):
def inner(executor, call):
@@ -94,6 +96,8 @@ class TestExecute:
_drain(executor.execute({}, ToolCall(id="c", name="x", arguments="{}"), "L"))
assert trace.spans[0].status == "error"
assert trace.spans[0].attributes["error.type"] == "RuntimeError"
assert trace.spans[0].error == "Tool call failed (RuntimeError)"
assert trace.spans[0].previews["error"] == "api down"
def test_nested_retrieval_parents_to_tool(self, trace):
def inner(executor, call):
@@ -109,6 +113,22 @@ class TestExecute:
assert retrieval.parent_id == tool.id
def test_tool_content_is_not_kept_when_capture_is_off(trace, monkeypatch):
monkeypatch.setattr(settings, "TRACES_CAPTURE_CONTENT", False)
def inner(executor, call):
executor.tool_calls.append(
{"tool_name": "t", "result": "secret tool output", "status": "error"}
)
return "", call.id
yield # pragma: no cover
_drain(_executor_with(inner).execute({}, ToolCall(id="c", name="x", arguments="{}"), "L"))
trace.finish()
record = trace.to_record()
assert "secret tool output" not in str(record)
class _Handler(LLMHandler):
def parse_response(self, response): # pragma: no cover - unused
raise NotImplementedError
@@ -169,7 +189,8 @@ class TestPausedCalls:
_drain(_Handler().handle_tool_calls(agent, [call], {"0": {"name": "telegram"}}, []))
(span,) = trace.spans
assert span.status == "denied"
assert span.error == "not allowed"
assert span.error is None
assert span.previews["error"] == "not allowed"
class TestContinuation:
@@ -227,7 +248,8 @@ class TestContinuation:
assert agent_span.status == "ok"
assert tool_span.parent_id == agent_span.id
assert tool_span.status == "denied"
assert tool_span.error == "too risky"
assert tool_span.error is None
assert tool_span.previews["error"] == "too risky"
def test_client_result_is_recorded(self, trace):
agent = self._agent()
+19
View File
@@ -353,3 +353,22 @@ class TestOutcome:
trace.outcome = "paused"
trace.finish(status="error")
assert trace.status == "error"
class TestRecordQuery:
def test_first_span_query_is_copied_into_summary(self):
trace = tracing.start_trace(source="search")
with tracing.activate(trace):
with tracing.span(tracing.KIND_RETRIEVAL, "retrieval") as s:
s.preview("query", "how do I deploy")
trace.finish()
assert trace.to_record()["summary"]["query"] == "how do I deploy"
def test_query_is_dropped_when_content_is_blocked(self):
trace = tracing.start_trace(source="search")
with tracing.activate(trace):
with tracing.span(tracing.KIND_RETRIEVAL, "retrieval") as s:
s.preview("query", "leaky")
tracing.mark_content_blocked()
trace.finish()
assert "query" not in trace.to_record()["summary"]