mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-14 10:17:55 +00:00
Keep tool output out of span errors and fix trace status and listing
Tool results, denial comments and tool exception text now reach a span only as a capture-gated preview; span.error is a fixed message, since it is stored and exported regardless of content settings. A turn whose stream yields an error event is recorded as failed. Search traces are listed by a query copied into the small summary column, so the Logs timeline never reads the spans JSONB. Adds GraphRAG span tests.
This commit is contained in:
1 parent
285fdc914c
commit
efec1a2f83
9 files changed
+134
-13
No files matched your search
@@ -132,6 +132,17 @@ class TestTraceLifecycle:
|
||||
assert any('"type": "error"' in s for s in stream)
|
||||
assert flushed[0].status == "error"
|
||||
|
||||
def test_yielded_error_marks_trace_error(self, flask_app, mock_mongo_db):
|
||||
"""A failed workflow node yields an error event instead of raising."""
|
||||
from docsgpt.api.answer.routes.base import BaseAnswerResource
|
||||
|
||||
with flask_app.app_context(), _captured_flushes() as flushed:
|
||||
_run(
|
||||
BaseAnswerResource(),
|
||||
_agent([{"type": "error", "error": "node failed"}]),
|
||||
)
|
||||
assert flushed[0].status == "error"
|
||||
|
||||
def test_abandoned_stream_still_flushes(self, flask_app, mock_mongo_db):
|
||||
from docsgpt.api.answer.routes.base import BaseAnswerResource
|
||||
|
||||
|
||||
@@ -167,7 +167,14 @@ class TestLogsTraceSummaries:
|
||||
assert row["trace"]["ref"] == {"field": "activity_id", "value": "act-1"}
|
||||
|
||||
def test_search_traces_are_listed(self, app, pg_conn):
|
||||
record = _trace(pg_conn, request_id=None, source="mcp", name="mcp", status="error")
|
||||
record = _trace(
|
||||
pg_conn,
|
||||
request_id=None,
|
||||
source="mcp",
|
||||
name="mcp",
|
||||
status="error",
|
||||
summary={"llm_calls": 0, "query": "how do I deploy"},
|
||||
)
|
||||
_trace(pg_conn, request_id=None, source="graph_extraction", name="graph_extraction Docs")
|
||||
search_rows = _logs(app, pg_conn, "owner", {"event_type": "search"}).json["logs"]
|
||||
assert len(search_rows) == 1
|
||||
|
||||
@@ -1021,3 +1021,41 @@ class TestPersonalizedPageRankWithoutScipy:
|
||||
docs = rag._get_data()
|
||||
|
||||
assert [doc["text"] for doc in docs] == ["near", "far"]
|
||||
|
||||
|
||||
class TestTraceSpans:
|
||||
"""GraphRAG searches and query embeddings are recorded in the execution trace."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _trace(self, monkeypatch):
|
||||
from docsgpt import tracing
|
||||
from docsgpt.core.settings import settings
|
||||
|
||||
monkeypatch.setattr(settings, "TRACES_ENABLED", True)
|
||||
monkeypatch.setattr(settings, "TRACES_CAPTURE_CONTENT", True)
|
||||
self.trace = tracing.start_trace(source="stream", capture_otel_context=False)
|
||||
with tracing.activate(self.trace):
|
||||
yield
|
||||
|
||||
def test_search_is_a_retrieval_span(self, _patch_llm_creator):
|
||||
rag = _make_retriever()
|
||||
docs = [{"text": "alpha", "title": "Doc A", "source": "a.md"}]
|
||||
with patch.object(rag, "_get_data", return_value=docs):
|
||||
assert rag.search("new question") == docs
|
||||
(span,) = [s for s in self.trace.spans if s.kind == "retrieval"]
|
||||
assert span.name == "retrieval GraphRAGRetriever"
|
||||
assert span.attributes["docsgpt.retriever"] == "GraphRAGRetriever"
|
||||
assert span.attributes["gen_ai.data_source.id"] == "src1"
|
||||
assert span.attributes["docsgpt.chunk_count"] == 1
|
||||
assert span.previews["chunks"][0]["title"] == "Doc A"
|
||||
|
||||
def test_embed_query_is_an_embedding_span(self):
|
||||
embedder = Mock()
|
||||
embedder.embed_query.return_value = [0.5]
|
||||
with patch(
|
||||
"docsgpt.retriever.graph_rag.get_embeddings", return_value=embedder
|
||||
):
|
||||
assert GraphRAGRetriever._embed_query(object(), "q") == [0.5]
|
||||
(span,) = self.trace.spans
|
||||
assert span.kind == "embedding"
|
||||
assert span.attributes["gen_ai.operation.name"] == "embeddings"
|
||||
@@ -82,7 +82,9 @@ class TestExecute:
|
||||
_drain(executor.execute({}, ToolCall(id="c", name="x", arguments="{}"), "L"))
|
||||
span = trace.spans[0]
|
||||
assert span.status == "error"
|
||||
assert span.error == "no such tool"
|
||||
# Tool output stays in the capture-gated preview, never in ``error``.
|
||||
assert span.error == "Tool call failed (ToolError)"
|
||||
assert span.previews["result"] == "no such tool"
|
||||
|
||||
def test_raised_error(self, trace):
|
||||
def inner(executor, call):
|
||||
@@ -94,6 +96,8 @@ class TestExecute:
|
||||
_drain(executor.execute({}, ToolCall(id="c", name="x", arguments="{}"), "L"))
|
||||
assert trace.spans[0].status == "error"
|
||||
assert trace.spans[0].attributes["error.type"] == "RuntimeError"
|
||||
assert trace.spans[0].error == "Tool call failed (RuntimeError)"
|
||||
assert trace.spans[0].previews["error"] == "api down"
|
||||
|
||||
def test_nested_retrieval_parents_to_tool(self, trace):
|
||||
def inner(executor, call):
|
||||
@@ -109,6 +113,22 @@ class TestExecute:
|
||||
assert retrieval.parent_id == tool.id
|
||||
|
||||
|
||||
def test_tool_content_is_not_kept_when_capture_is_off(trace, monkeypatch):
|
||||
monkeypatch.setattr(settings, "TRACES_CAPTURE_CONTENT", False)
|
||||
|
||||
def inner(executor, call):
|
||||
executor.tool_calls.append(
|
||||
{"tool_name": "t", "result": "secret tool output", "status": "error"}
|
||||
)
|
||||
return "", call.id
|
||||
yield # pragma: no cover
|
||||
|
||||
_drain(_executor_with(inner).execute({}, ToolCall(id="c", name="x", arguments="{}"), "L"))
|
||||
trace.finish()
|
||||
record = trace.to_record()
|
||||
assert "secret tool output" not in str(record)
|
||||
|
||||
|
||||
class _Handler(LLMHandler):
|
||||
def parse_response(self, response): # pragma: no cover - unused
|
||||
raise NotImplementedError
|
||||
@@ -169,7 +189,8 @@ class TestPausedCalls:
|
||||
_drain(_Handler().handle_tool_calls(agent, [call], {"0": {"name": "telegram"}}, []))
|
||||
(span,) = trace.spans
|
||||
assert span.status == "denied"
|
||||
assert span.error == "not allowed"
|
||||
assert span.error is None
|
||||
assert span.previews["error"] == "not allowed"
|
||||
|
||||
|
||||
class TestContinuation:
|
||||
@@ -227,7 +248,8 @@ class TestContinuation:
|
||||
assert agent_span.status == "ok"
|
||||
assert tool_span.parent_id == agent_span.id
|
||||
assert tool_span.status == "denied"
|
||||
assert tool_span.error == "too risky"
|
||||
assert tool_span.error is None
|
||||
assert tool_span.previews["error"] == "too risky"
|
||||
|
||||
def test_client_result_is_recorded(self, trace):
|
||||
agent = self._agent()
|
||||
|
||||
@@ -353,3 +353,22 @@ class TestOutcome:
|
||||
trace.outcome = "paused"
|
||||
trace.finish(status="error")
|
||||
assert trace.status == "error"
|
||||
|
||||
|
||||
class TestRecordQuery:
|
||||
def test_first_span_query_is_copied_into_summary(self):
|
||||
trace = tracing.start_trace(source="search")
|
||||
with tracing.activate(trace):
|
||||
with tracing.span(tracing.KIND_RETRIEVAL, "retrieval") as s:
|
||||
s.preview("query", "how do I deploy")
|
||||
trace.finish()
|
||||
assert trace.to_record()["summary"]["query"] == "how do I deploy"
|
||||
|
||||
def test_query_is_dropped_when_content_is_blocked(self):
|
||||
trace = tracing.start_trace(source="search")
|
||||
with tracing.activate(trace):
|
||||
with tracing.span(tracing.KIND_RETRIEVAL, "retrieval") as s:
|
||||
s.preview("query", "leaky")
|
||||
tracing.mark_content_blocked()
|
||||
trace.finish()
|
||||
assert "query" not in trace.to_record()["summary"]
|
||||
Reference in new issue
Block a user