mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-13 18:15:02 +00:00
test(v1): cover structured-output, reasoning-leak, multimodal and stateless continuation
Raise patch coverage on the new v1 code paths: - translator: content_to_text (text/image/string parts), extract_response_schema (response_format json_schema/json_object + response_schema shapes), _split_leaked_reasoning (single/multi/double-quoted/brace-in-value), translate_request structured-output + sampling + multimodal surfacing (normal and continuation branches), and translate_response / translate_stream_event reasoning-leak gating (answer + structured_answer). - stream_processor: build_continuation_from_messages — pending-call rebuild, invalid/dict argument handling, last-tool-call selection, prior-message slice, and the no-tool-calls error. translator patch coverage 54% -> ~100% of changed lines; stream_processor continuation path fully covered.
This commit is contained in:
1 parent
b212c5cedd
commit
45cae8f6b8
2 files changed
+522
No files matched your search
@@ -177,5 +177,142 @@ class TestToolPreFetch:
|
||||
"""Tests for tool pre-fetching with saved parameter values."""
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestBuildContinuationFromMessages:
|
||||
"""Stateless tool continuation rebuilt from the resent messages array.
|
||||
|
||||
OpenAI-compatible clients (opencode, etc.) resend the full conversation but
|
||||
carry no conversation_id, so the agent + pending tool calls are reconstructed
|
||||
directly from the request instead of from server-side ``pending_tool_state``.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _make_processor():
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from application.api.answer.services.stream_processor import StreamProcessor
|
||||
|
||||
processor = StreamProcessor({"question": ""}, {"sub": "user_123"})
|
||||
fake_agent = MagicMock()
|
||||
fake_agent.tool_executor.get_tools.return_value = {"search": object()}
|
||||
# build_agent touches the DB / config; stub it for this unit test.
|
||||
processor.build_agent = MagicMock(return_value=fake_agent)
|
||||
return processor, fake_agent
|
||||
|
||||
def _messages_with_tool_call(self, arguments):
|
||||
return [
|
||||
{"role": "system", "content": "You are a bot"},
|
||||
{"role": "user", "content": "search the docs"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "search", "arguments": arguments},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "found it"},
|
||||
]
|
||||
|
||||
def test_rebuilds_pending_tool_calls_and_prior_messages(self):
|
||||
processor, fake_agent = self._make_processor()
|
||||
messages = self._messages_with_tool_call('{"q": "docs"}')
|
||||
tool_actions = [{"call_id": "call_1", "result": "found it"}]
|
||||
|
||||
(
|
||||
agent,
|
||||
prior_messages,
|
||||
tools_dict,
|
||||
pending_tool_calls,
|
||||
returned_actions,
|
||||
reasoning,
|
||||
) = processor.build_continuation_from_messages(messages, tool_actions)
|
||||
|
||||
assert agent is fake_agent
|
||||
assert tools_dict == fake_agent.tool_executor.get_tools.return_value
|
||||
assert returned_actions is tool_actions
|
||||
assert reasoning == ""
|
||||
# prior_messages stop before the assistant-with-tool_calls message.
|
||||
assert prior_messages == messages[:2]
|
||||
assert len(pending_tool_calls) == 1
|
||||
call = pending_tool_calls[0]
|
||||
assert call["call_id"] == "call_1"
|
||||
assert call["name"] == "search"
|
||||
assert call["tool_name"] == "search"
|
||||
assert call["action_name"] == "search"
|
||||
assert call["arguments"] == {"q": "docs"}
|
||||
processor.build_agent.assert_called_once_with("")
|
||||
|
||||
def test_invalid_json_arguments_default_to_empty_dict(self):
|
||||
processor, _ = self._make_processor()
|
||||
messages = self._messages_with_tool_call("not-json")
|
||||
|
||||
_, _, _, pending_tool_calls, _, _ = (
|
||||
processor.build_continuation_from_messages(messages, [])
|
||||
)
|
||||
|
||||
assert pending_tool_calls[0]["arguments"] == {}
|
||||
|
||||
def test_dict_arguments_passed_through(self):
|
||||
processor, _ = self._make_processor()
|
||||
messages = self._messages_with_tool_call({"already": "parsed"})
|
||||
|
||||
_, _, _, pending_tool_calls, _, _ = (
|
||||
processor.build_continuation_from_messages(messages, [])
|
||||
)
|
||||
|
||||
assert pending_tool_calls[0]["arguments"] == {"already": "parsed"}
|
||||
|
||||
def test_uses_last_assistant_tool_call_message(self):
|
||||
processor, _ = self._make_processor()
|
||||
messages = [
|
||||
{"role": "user", "content": "first"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "old",
|
||||
"type": "function",
|
||||
"function": {"name": "search", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "old", "content": "r1"},
|
||||
{"role": "user", "content": "second"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "new",
|
||||
"type": "function",
|
||||
"function": {"name": "search", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "new", "content": "r2"},
|
||||
]
|
||||
|
||||
_, prior_messages, _, pending_tool_calls, _, _ = (
|
||||
processor.build_continuation_from_messages(messages, [])
|
||||
)
|
||||
|
||||
assert pending_tool_calls[0]["call_id"] == "new"
|
||||
# Everything up to (not including) the last assistant tool_calls message.
|
||||
assert prior_messages == messages[:4]
|
||||
|
||||
def test_raises_when_no_assistant_tool_calls(self):
|
||||
processor, _ = self._make_processor()
|
||||
messages = [
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "assistant", "content": "no tools here"},
|
||||
]
|
||||
|
||||
with pytest.raises(ValueError, match="No assistant message with tool_calls"):
|
||||
processor.build_continuation_from_messages(messages, [])
|
||||
|
||||
|
||||
@@ -10,7 +10,11 @@ import pytest
|
||||
|
||||
from application.api.v1.translator import (
|
||||
_get_client_tool_name,
|
||||
_split_leaked_reasoning,
|
||||
_strip_repr_quotes,
|
||||
content_to_text,
|
||||
convert_history,
|
||||
extract_response_schema,
|
||||
extract_system_prompt,
|
||||
extract_tool_results,
|
||||
is_continuation,
|
||||
@@ -708,3 +712,384 @@ class TestGetClientToolName:
|
||||
|
||||
def test_returns_empty_when_no_fields(self):
|
||||
assert _get_client_tool_name({}) == ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# content_to_text
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestContentToText:
|
||||
|
||||
def test_plain_string_passthrough(self):
|
||||
assert content_to_text("hello") == "hello"
|
||||
|
||||
def test_none_returns_empty(self):
|
||||
assert content_to_text(None) == ""
|
||||
|
||||
def test_text_parts_joined(self):
|
||||
content = [
|
||||
{"type": "text", "text": "line one"},
|
||||
{"type": "text", "text": "line two"},
|
||||
]
|
||||
assert content_to_text(content) == "line one\nline two"
|
||||
|
||||
def test_image_parts_dropped(self):
|
||||
content = [
|
||||
{"type": "text", "text": "describe this"},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/png;base64,xxx"}},
|
||||
]
|
||||
# Only the text part contributes; the image is dropped from the flattened text.
|
||||
assert content_to_text(content) == "describe this"
|
||||
|
||||
def test_bare_string_parts_included(self):
|
||||
assert content_to_text(["a", "b"]) == "a\nb"
|
||||
|
||||
def test_missing_text_field_becomes_empty(self):
|
||||
assert content_to_text([{"type": "text"}]) == ""
|
||||
|
||||
def test_non_string_non_list_coerced(self):
|
||||
assert content_to_text(123) == "123"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# extract_response_schema
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
_SCHEMA = {
|
||||
"type": "object",
|
||||
"properties": {"answer": {"type": "string"}},
|
||||
"required": ["answer"],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestExtractResponseSchema:
|
||||
|
||||
def test_none_when_absent(self):
|
||||
assert extract_response_schema({}) is None
|
||||
|
||||
def test_response_format_json_schema_wrapper(self):
|
||||
data = {
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
"json_schema": {"name": "ans", "schema": _SCHEMA},
|
||||
}
|
||||
}
|
||||
assert extract_response_schema(data) == _SCHEMA
|
||||
|
||||
def test_response_format_bare_schema_under_json_schema(self):
|
||||
# A bare schema (no "schema" wrapper) but with a top-level "type" is tolerated.
|
||||
data = {
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
"json_schema": _SCHEMA,
|
||||
}
|
||||
}
|
||||
assert extract_response_schema(data) == _SCHEMA
|
||||
|
||||
def test_response_format_json_object_carries_no_schema(self):
|
||||
data = {"response_format": {"type": "json_object"}}
|
||||
assert extract_response_schema(data) is None
|
||||
|
||||
def test_response_schema_raw_object(self):
|
||||
assert extract_response_schema({"response_schema": _SCHEMA}) == _SCHEMA
|
||||
|
||||
def test_response_schema_wrapper(self):
|
||||
data = {"response_schema": {"schema": _SCHEMA}}
|
||||
assert extract_response_schema(data) == _SCHEMA
|
||||
|
||||
def test_response_schema_takes_precedence_over_response_format(self):
|
||||
other = {"type": "object", "properties": {}}
|
||||
data = {
|
||||
"response_schema": _SCHEMA,
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
"json_schema": {"schema": other},
|
||||
},
|
||||
}
|
||||
assert extract_response_schema(data) == _SCHEMA
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _split_leaked_reasoning
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestSplitLeakedReasoning:
|
||||
|
||||
def test_no_marker_is_noop(self):
|
||||
assert _split_leaked_reasoning("just an answer") == ("just an answer", "")
|
||||
|
||||
def test_none_is_noop(self):
|
||||
assert _split_leaked_reasoning(None) == (None, "")
|
||||
|
||||
def test_extracts_single_thought(self):
|
||||
content = "{'type': 'thought', 'thought': 'let me think'}The answer is 42."
|
||||
clean, leaked = _split_leaked_reasoning(content)
|
||||
assert clean == "The answer is 42."
|
||||
assert leaked == "let me think"
|
||||
|
||||
def test_extracts_multiple_thoughts(self):
|
||||
content = (
|
||||
"{'type': 'thought', 'thought': 'first'}"
|
||||
"partial"
|
||||
"{'type': 'thought', 'thought': 'second'}done"
|
||||
)
|
||||
clean, leaked = _split_leaked_reasoning(content)
|
||||
assert clean == "partialdone"
|
||||
assert leaked == "firstsecond"
|
||||
|
||||
def test_double_quoted_thought_value(self):
|
||||
# When the token contains an apostrophe the repr uses double quotes.
|
||||
content = "{'type': 'thought', 'thought': \"I'll check\"}answer"
|
||||
clean, leaked = _split_leaked_reasoning(content)
|
||||
assert clean == "answer"
|
||||
assert leaked == "I'll check"
|
||||
|
||||
def test_thought_value_with_brace_not_truncated(self):
|
||||
content = "{'type': 'thought', 'thought': 'use {json} here'}final"
|
||||
clean, leaked = _split_leaked_reasoning(content)
|
||||
assert clean == "final"
|
||||
assert leaked == "use {json} here"
|
||||
|
||||
def test_strip_repr_quotes_unquoted_passthrough(self):
|
||||
# Defensive branch: an unquoted value is returned unchanged.
|
||||
assert _strip_repr_quotes("plain") == "plain"
|
||||
assert _strip_repr_quotes("'quoted'") == "quoted"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# translate_request — structured outputs / sampling / multimodal
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestTranslateRequestStructuredOutputs:
|
||||
|
||||
def test_response_format_surfaces_json_schema_strict_default(self):
|
||||
data = {
|
||||
"messages": [{"role": "user", "content": "Hi"}],
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
"json_schema": {"name": "ans", "schema": _SCHEMA},
|
||||
},
|
||||
}
|
||||
result = translate_request(data, "key")
|
||||
assert result["json_schema"] == _SCHEMA
|
||||
assert result["json_schema_strict"] is True
|
||||
|
||||
def test_response_format_honours_explicit_strict_false(self):
|
||||
data = {
|
||||
"messages": [{"role": "user", "content": "Hi"}],
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
"json_schema": {"name": "ans", "schema": _SCHEMA, "strict": False},
|
||||
},
|
||||
}
|
||||
result = translate_request(data, "key")
|
||||
assert result["json_schema_strict"] is False
|
||||
|
||||
def test_json_object_mode_flag(self):
|
||||
data = {
|
||||
"messages": [{"role": "user", "content": "Hi"}],
|
||||
"response_format": {"type": "json_object"},
|
||||
}
|
||||
result = translate_request(data, "key")
|
||||
assert result["json_object"] is True
|
||||
assert "json_schema" not in result
|
||||
|
||||
def test_sampling_params_forwarded(self):
|
||||
data = {
|
||||
"messages": [{"role": "user", "content": "Hi"}],
|
||||
"temperature": 0.2,
|
||||
"top_p": 0.9,
|
||||
"seed": 7,
|
||||
}
|
||||
result = translate_request(data, "key")
|
||||
assert result["llm_params"] == {"temperature": 0.2, "top_p": 0.9, "seed": 7}
|
||||
|
||||
def test_max_tokens_alias_dropped_when_canonical_present(self):
|
||||
data = {
|
||||
"messages": [{"role": "user", "content": "Hi"}],
|
||||
"max_tokens": 100,
|
||||
"max_completion_tokens": 200,
|
||||
}
|
||||
result = translate_request(data, "key")
|
||||
assert result["llm_params"]["max_completion_tokens"] == 200
|
||||
assert "max_tokens" not in result["llm_params"]
|
||||
|
||||
def test_multimodal_content_preserved_and_question_flattened(self):
|
||||
content = [
|
||||
{"type": "text", "text": "what is this"},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/png;base64,xxx"}},
|
||||
]
|
||||
data = {"messages": [{"role": "user", "content": content}]}
|
||||
result = translate_request(data, "key")
|
||||
assert result["question"] == "what is this"
|
||||
assert result["multimodal_content"] == content
|
||||
|
||||
def test_plain_text_request_has_no_multimodal_content(self):
|
||||
data = {"messages": [{"role": "user", "content": "plain"}]}
|
||||
result = translate_request(data, "key")
|
||||
assert "multimodal_content" not in result
|
||||
|
||||
def test_continuation_forwards_schema_and_sampling(self):
|
||||
data = {
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hi"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "search", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "result"},
|
||||
],
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
"json_schema": {"name": "ans", "schema": _SCHEMA},
|
||||
},
|
||||
"temperature": 0.5,
|
||||
}
|
||||
result = translate_request(data, "key")
|
||||
assert result["tool_actions"]
|
||||
assert result["json_schema"] == _SCHEMA
|
||||
assert result["json_schema_strict"] is True
|
||||
assert result["llm_params"] == {"temperature": 0.5}
|
||||
|
||||
def test_continuation_forwards_tools_and_json_object(self):
|
||||
data = {
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hi"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "search", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "result"},
|
||||
],
|
||||
"tools": [{"type": "function", "function": {"name": "search"}}],
|
||||
"response_format": {"type": "json_object"},
|
||||
}
|
||||
result = translate_request(data, "key")
|
||||
assert result["client_tools"] == data["tools"]
|
||||
assert result["json_object"] is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# translate_response / translate_stream_event — reasoning-leak handling
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestReasoningLeakHandling:
|
||||
|
||||
def test_response_strips_leak_only_when_enabled(self):
|
||||
answer = "{'type': 'thought', 'thought': 'hmm'}Final answer."
|
||||
result = translate_response(
|
||||
conversation_id="",
|
||||
answer=answer,
|
||||
sources=None,
|
||||
tool_calls=None,
|
||||
thought="",
|
||||
model_name="agent",
|
||||
strip_reasoning_leak=True,
|
||||
)
|
||||
msg = result["choices"][0]["message"]
|
||||
assert msg["content"] == "Final answer."
|
||||
assert msg["reasoning_content"] == "hmm"
|
||||
|
||||
def test_response_preserves_content_when_strip_disabled(self):
|
||||
answer = "{'type': 'thought', 'thought': 'hmm'}Final answer."
|
||||
result = translate_response(
|
||||
conversation_id="",
|
||||
answer=answer,
|
||||
sources=None,
|
||||
tool_calls=None,
|
||||
thought="",
|
||||
model_name="agent",
|
||||
strip_reasoning_leak=False,
|
||||
)
|
||||
msg = result["choices"][0]["message"]
|
||||
# Untouched: the leak marker is left in content verbatim.
|
||||
assert msg["content"] == answer
|
||||
assert "reasoning_content" not in msg
|
||||
|
||||
def test_response_combines_thought_and_leaked_reasoning(self):
|
||||
answer = "{'type': 'thought', 'thought': 'leaked'}Answer."
|
||||
result = translate_response(
|
||||
conversation_id="",
|
||||
answer=answer,
|
||||
sources=None,
|
||||
tool_calls=None,
|
||||
thought="explicit ",
|
||||
model_name="agent",
|
||||
strip_reasoning_leak=True,
|
||||
)
|
||||
msg = result["choices"][0]["message"]
|
||||
assert msg["reasoning_content"] == "explicit leaked"
|
||||
|
||||
def test_stream_answer_splits_leak_when_enabled(self):
|
||||
chunks = translate_stream_event(
|
||||
{"type": "answer", "answer": "{'type': 'thought', 'thought': 'r'}hi"},
|
||||
"chatcmpl-1", "agent", True,
|
||||
)
|
||||
deltas = [
|
||||
json.loads(c.replace("data: ", "").strip())["choices"][0]["delta"]
|
||||
for c in chunks
|
||||
]
|
||||
assert {"reasoning_content": "r"} in deltas
|
||||
assert {"content": "hi"} in deltas
|
||||
|
||||
def test_stream_answer_preserves_content_when_disabled(self):
|
||||
raw = "{'type': 'thought', 'thought': 'r'}hi"
|
||||
chunks = translate_stream_event(
|
||||
{"type": "answer", "answer": raw}, "chatcmpl-1", "agent", False,
|
||||
)
|
||||
deltas = [
|
||||
json.loads(c.replace("data: ", "").strip())["choices"][0]["delta"]
|
||||
for c in chunks
|
||||
]
|
||||
assert {"content": raw} in deltas
|
||||
assert all("reasoning_content" not in d for d in deltas)
|
||||
|
||||
def test_stream_structured_answer_event(self):
|
||||
chunks = translate_stream_event(
|
||||
{"type": "structured_answer", "answer": '{"answer": "42"}'},
|
||||
"chatcmpl-1", "agent", True,
|
||||
)
|
||||
deltas = [
|
||||
json.loads(c.replace("data: ", "").strip())["choices"][0]["delta"]
|
||||
for c in chunks
|
||||
]
|
||||
assert {"content": '{"answer": "42"}'} in deltas
|
||||
|
||||
def test_stream_structured_answer_splits_leak(self):
|
||||
chunks = translate_stream_event(
|
||||
{
|
||||
"type": "structured_answer",
|
||||
"answer": "{'type': 'thought', 'thought': 'why'}{\"answer\": \"42\"}",
|
||||
},
|
||||
"chatcmpl-1", "agent", True,
|
||||
)
|
||||
deltas = [
|
||||
json.loads(c.replace("data: ", "").strip())["choices"][0]["delta"]
|
||||
for c in chunks
|
||||
]
|
||||
assert {"reasoning_content": "why"} in deltas
|
||||
assert {"content": '{"answer": "42"}'} in deltas
|
||||
Reference in new issue
Block a user