From 472918f577c8f2007f9e1266ece02d603aea7072 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 4 Jun 2026 11:51:13 +0100 Subject: [PATCH] feat(v1): support response_format / response_schema on /v1/chat/completions Make the OpenAI-compatible Chat Completions endpoint honor per-request Structured Outputs and keep it OpenAI-compatible. - translate_request now forwards the request's `response_format` (json_schema) or a `response_schema` convenience field to the agent as its json_schema, overriding the agent-configured schema for that request. - Honor `response_format.json_schema.strict` (default true); strict:false passes the schema through without forcing additionalProperties:false / all-required (OpenAI's lenient mode). - Support `response_format {"type":"json_object"}` via the provider's native JSON mode. - Keep `content` clean: some models echo their reasoning into content as stringified `{'type': 'thought', ...}` reprs when response_format is set. Strip those from content and reroute them to `reasoning_content` (OpenAI never puts reasoning in content), for both streaming and non-streaming. Flow: translate_request -> StreamProcessor._configure_agent -> Agent.json_schema / json_schema_strict / json_object -> _llm_gen -> prepare_structured_output_format(schema, strict). --- application/agents/base.py | 17 ++- .../api/answer/services/stream_processor.py | 14 ++ application/api/v1/translator.py | 121 ++++++++++++++++-- application/llm/google_ai.py | 8 +- application/llm/openai.py | 11 +- 5 files changed, 154 insertions(+), 17 deletions(-) diff --git a/application/agents/base.py b/application/agents/base.py index e887c22d..b9d41109 100644 --- a/application/agents/base.py +++ b/application/agents/base.py @@ -33,6 +33,8 @@ class BaseAgent(ABC): decoded_token: Optional[Dict] = None, attachments: Optional[List[Dict]] = None, json_schema: Optional[Dict] = None, + json_schema_strict: bool = True, + json_object: bool = False, limited_token_mode: Optional[bool] = False, token_limit: Optional[int] = settings.DEFAULT_AGENT_LIMITS["token_limit"], limited_request_mode: Optional[bool] = False, @@ -108,6 +110,11 @@ class BaseAgent(ABC): self.json_schema = normalize_json_schema_payload(json_schema) except JsonSchemaValidationError as exc: logger.warning("Ignoring invalid JSON schema payload: %s", exc) + # Per-request structured-output controls (OpenAI-compatible): + # ``json_schema_strict`` mirrors response_format.json_schema.strict; + # ``json_object`` mirrors response_format {"type":"json_object"}. + self.json_schema_strict = json_schema_strict + self.json_object = json_object self.limited_token_mode = limited_token_mode self.token_limit = token_limit self.limited_request_mode = limited_request_mode @@ -572,13 +579,21 @@ class BaseAgent(ABC): and self.llm._supports_structured_output() ): structured_format = self.llm.prepare_structured_output_format( - self.json_schema + self.json_schema, strict=getattr(self, "json_schema_strict", True) ) if structured_format: if self.llm_name == "openai": gen_kwargs["response_format"] = structured_format elif self.llm_name == "google": gen_kwargs["response_schema"] = structured_format + elif ( + getattr(self, "json_object", False) + and self.llm_name == "openai" + and hasattr(self.llm, "_supports_structured_output") + and self.llm._supports_structured_output() + ): + # OpenAI json_object mode: guarantee valid JSON, no schema enforcement. + gen_kwargs["response_format"] = {"type": "json_object"} if ( settings.OPENAI_RESPONSES_STORE and hasattr(self.llm, "_uses_responses_api") diff --git a/application/api/answer/services/stream_processor.py b/application/api/answer/services/stream_processor.py index e923b26a..dfaf5b9d 100644 --- a/application/api/answer/services/stream_processor.py +++ b/application/api/answer/services/stream_processor.py @@ -647,6 +647,18 @@ class StreamProcessor: } ) + # Per-request structured output: a ``response_format`` / ``response_schema`` + # in the request (surfaced by the v1 translator as ``json_schema``) overrides + # the agent's configured schema for this call. Invalid schemas are ignored + # downstream by the agent (normalize_json_schema_payload). + request_json_schema = self.data.get("json_schema") + if request_json_schema is not None: + self.agent_config["json_schema"] = request_json_schema + if self.data.get("json_schema_strict") is not None: + self.agent_config["json_schema_strict"] = self.data.get("json_schema_strict") + if self.data.get("json_object"): + self.agent_config["json_object"] = True + def _configure_retriever(self): """Assemble retriever config; agent's values are authoritative when bound.""" # BYOM scope: owner for shared-agent BYOM, caller for own BYOM, @@ -1254,6 +1266,8 @@ class StreamProcessor: "decoded_token": self.decoded_token, "attachments": self.attachments, "json_schema": self.agent_config.get("json_schema"), + "json_schema_strict": self.agent_config.get("json_schema_strict", True), + "json_object": self.agent_config.get("json_object", False), "compressed_summary": self.compressed_summary, "llm": llm, "llm_handler": llm_handler, diff --git a/application/api/v1/translator.py b/application/api/v1/translator.py index a4d48f0c..d7c91898 100644 --- a/application/api/v1/translator.py +++ b/application/api/v1/translator.py @@ -7,9 +7,49 @@ This module handles: """ import json +import re import time from typing import Any, Dict, List, Optional +# Some upstream models/proxies echo their reasoning into ``content`` as +# stringified ``{'type': 'thought', 'thought': '...'}`` event reprs (instead of +# using the separate reasoning channel) — most visibly when ``response_format`` +# is set. OpenAI's API never puts reasoning in ``content``, so for the +# OpenAI-compatible endpoint we strip these and reroute them to +# ``reasoning_content`` to keep ``content`` clean and compatible. +# The thought value is a Python string repr: single-quoted, or double-quoted when +# the token contains an apostrophe (e.g. "'ll"). Match the full quoted value +# (honoring escapes) so tokens containing ``}`` or newlines don't truncate the +# match and leave stray ``'}`` tails in the content. +_LEAKED_THOUGHT_RE = re.compile( + r"""\{'type': 'thought', 'thought': ('(?:[^'\\]|\\.)*'|"(?:[^"\\]|\\.)*")\}""", + re.DOTALL, +) + + +def _strip_repr_quotes(value: str) -> str: + value = value.strip() + if len(value) >= 2 and value[0] in "\"'" and value[-1] == value[0]: + return value[1:-1] + return value + + +def _split_leaked_reasoning(content: Optional[str]) -> tuple: + """Return ``(clean_content, leaked_reasoning)``. + + ``clean_content`` has any stringified thought-event reprs removed; + ``leaked_reasoning`` is the concatenated reasoning text that was extracted. + A no-op (returns the input unchanged) when no leak markers are present. + """ + if not content or "'type': 'thought'" not in content: + return content, "" + extracted: List[str] = [] + cleaned = _LEAKED_THOUGHT_RE.sub( + lambda m: (extracted.append(_strip_repr_quotes(m.group(1))) or ""), content + ) + return cleaned, "".join(extracted) + + def _get_client_tool_name(tc: Dict) -> str: """Return the original tool name for client-facing responses. @@ -121,6 +161,37 @@ def convert_history(messages: List[Dict]) -> List[Dict]: return history +def extract_response_schema(data: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """Extract a JSON schema for structured output from a chat-completions request. + + Supports two request shapes: + - OpenAI ``response_format``: + ``{"type": "json_schema", "json_schema": {"name": ..., "schema": {...}}}`` + (a bare schema under ``json_schema`` is also tolerated). + - ``response_schema`` convenience field: a raw JSON Schema object, or a + ``{"schema": {...}}`` wrapper. + + Returns a raw JSON Schema object, or None. ``response_format`` + ``{"type": "json_object"}`` carries no schema to enforce and yields None + (the model is still steered by the system prompt). + """ + response_schema = data.get("response_schema") + if isinstance(response_schema, dict) and response_schema: + inner = response_schema.get("schema") + return inner if isinstance(inner, dict) else response_schema + + response_format = data.get("response_format") + if isinstance(response_format, dict) and response_format.get("type") == "json_schema": + json_schema = response_format.get("json_schema") + if isinstance(json_schema, dict): + schema = json_schema.get("schema") + if isinstance(schema, dict): + return schema + if "type" in json_schema: + return json_schema + return None + + def translate_request( data: Dict[str, Any], api_key: str ) -> Dict[str, Any]: @@ -134,6 +205,12 @@ def translate_request( Dict suitable for passing to ``StreamProcessor``. """ messages = data.get("messages", []) + response_schema = extract_response_schema(data) + _rf = data.get("response_format") + _rf = _rf if isinstance(_rf, dict) else {} + # OpenAI Structured Outputs default to strict; honor an explicit strict:false. + json_schema_strict = bool((_rf.get("json_schema") or {}).get("strict", True)) + json_object_mode = _rf.get("type") == "json_object" # Check for continuation (tool results after assistant tool_calls) if is_continuation(messages): @@ -155,6 +232,11 @@ def translate_request( # Carry tools forward for next iteration if data.get("tools"): result["client_tools"] = data["tools"] + if response_schema is not None: + result["json_schema"] = response_schema + result["json_schema_strict"] = json_schema_strict + if json_object_mode: + result["json_object"] = True return result # Normal request — extract question from last user message @@ -189,6 +271,12 @@ def translate_request( if docsgpt.get("attachments"): result["attachments"] = docsgpt["attachments"] + if response_schema is not None: + result["json_schema"] = response_schema + result["json_schema_strict"] = json_schema_strict + if json_object_mode: + result["json_object"] = True + return result @@ -246,9 +334,11 @@ def translate_response( ] finish_reason = "tool_calls" else: - message["content"] = answer - if thought: - message["reasoning_content"] = thought + clean_answer, leaked_reasoning = _split_leaked_reasoning(answer) + message["content"] = clean_answer + combined_reasoning = (thought or "") + leaked_reasoning + if combined_reasoning: + message["reasoning_content"] = combined_reasoning finish_reason = "stop" result: Dict[str, Any] = { @@ -341,9 +431,15 @@ def translate_stream_event( chunks: List[str] = [] if event_type == "answer": - chunks.append( - _make_chunk(completion_id, model_name, {"content": event_data.get("answer", "")}) - ) + clean, leaked = _split_leaked_reasoning(event_data.get("answer", "")) + if leaked: + chunks.append( + _make_chunk(completion_id, model_name, {"reasoning_content": leaked}) + ) + if clean: + chunks.append( + _make_chunk(completion_id, model_name, {"content": clean}) + ) elif event_type == "thought": chunks.append( @@ -427,12 +523,15 @@ def translate_stream_event( chunks.append(f"data: {json.dumps(error_data)}\n\n") elif event_type == "structured_answer": - chunks.append( - _make_chunk( - completion_id, model_name, - {"content": event_data.get("answer", "")}, + clean, leaked = _split_leaked_reasoning(event_data.get("answer", "")) + if leaked: + chunks.append( + _make_chunk(completion_id, model_name, {"reasoning_content": leaked}) + ) + if clean: + chunks.append( + _make_chunk(completion_id, model_name, {"content": clean}) ) - ) # Skip: tool_calls (redundant), research_plan, research_progress diff --git a/application/llm/google_ai.py b/application/llm/google_ai.py index aca099c5..e1fcf7bc 100644 --- a/application/llm/google_ai.py +++ b/application/llm/google_ai.py @@ -653,8 +653,12 @@ class GoogleLLM(BaseLLM): """Return whether this LLM supports structured JSON output.""" return True - def prepare_structured_output_format(self, json_schema): - """Convert JSON schema to Google AI structured output format.""" + def prepare_structured_output_format(self, json_schema, strict=True): + """Convert JSON schema to Google AI structured output format. + + ``strict`` is accepted for signature parity with the OpenAI provider; + Google enforces the schema natively via ``response_schema``. + """ if not json_schema: return None type_map = { diff --git a/application/llm/openai.py b/application/llm/openai.py index e3b9000e..1ae59632 100644 --- a/application/llm/openai.py +++ b/application/llm/openai.py @@ -879,7 +879,7 @@ class OpenAILLM(BaseLLM): if effort: kwargs["reasoning_effort"] = effort - def prepare_structured_output_format(self, json_schema): + def prepare_structured_output_format(self, json_schema, strict=True): if not json_schema: return None try: @@ -914,7 +914,12 @@ class OpenAILLM(BaseLLM): return schema_copy return schema_obj - processed_schema = add_additional_properties_false(json_schema) + # Strict mode requires additionalProperties:false + all-required on every + # object (OpenAI Structured Outputs). When strict is false (OpenAI's + # lenient json_schema), pass the schema through unchanged. + processed_schema = ( + add_additional_properties_false(json_schema) if strict else json_schema + ) result = { "type": "json_schema", @@ -924,7 +929,7 @@ class OpenAILLM(BaseLLM): "description", "Structured response" ), "schema": processed_schema, - "strict": True, + "strict": strict, }, }