From 142a9201f4aed3224553eabff76fb670353f427a Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 3 Jun 2026 13:43:04 +0100 Subject: [PATCH 1/3] feat: add OpenAI Responses API support via per-model api_flavor Route opted-in models to /v1/responses while keeping Chat Completions as the default. The flavor is selected per-model in the catalog YAML and threaded through ModelCapabilities (with a new reasoning_effort field). - Translate chat-shaped messages/tools/structured-output into the Responses input / text.format / flattened-tool shapes at the provider boundary; stored history and intra-turn tool round-trips need no migration. - Normalize the typed SSE event stream back into the existing handler contract (content / thought / tool-call chunks) so the streaming accumulator and OpenAI handler stay unchanged. - Carry encrypted reasoning items across the in-turn tool loop (provider-local, keyed by call_id) for stateless reasoning continuity. - OPENAI_RESPONSES_STORE opt-in enables server-side state and previous_response_id chaining across turns, persisted in message metadata. - Set gpt-5.5 to api_flavor: responses, reasoning_effort: medium. - Remove AzureOpenAILLM and the azure_openai provider/catalog. --- application/agents/base.py | 29 ++ application/core/model_settings.py | 9 +- application/core/model_yaml.py | 33 ++ application/core/models/azure_openai.yaml | 31 -- application/core/models/openai.yaml | 2 + application/core/settings.py | 7 + application/llm/openai.py | 489 +++++++++++++++++++++- application/llm/providers/__init__.py | 2 - application/llm/providers/azure_openai.py | 30 -- tests/core/test_model_registry_yaml.py | 19 - tests/core/test_model_settings.py | 1 - tests/core/test_model_utils.py | 10 - tests/llm/test_openai.py | 39 -- tests/llm/test_openai_responses.py | 366 ++++++++++++++++ 14 files changed, 917 insertions(+), 150 deletions(-) delete mode 100644 application/core/models/azure_openai.yaml delete mode 100644 application/llm/providers/azure_openai.py create mode 100644 tests/llm/test_openai_responses.py diff --git a/application/agents/base.py b/application/agents/base.py index 22b808d9..e887c22d 100644 --- a/application/agents/base.py +++ b/application/agents/base.py @@ -123,6 +123,26 @@ class BaseAgent(ABC): self, query: str, log_context: LogContext = None ) -> Generator[Dict, None, None]: yield from self._gen_inner(query, log_context) + yield from self._emit_responses_metadata() + + def _emit_responses_metadata(self) -> Generator[Dict, None, None]: + """Surface the latest Responses API id so the route can persist it in + message metadata for previous_response_id chaining across turns.""" + if not settings.OPENAI_RESPONSES_STORE: + return + response_id = getattr(self.llm, "_last_response_id", None) + if response_id: + yield {"metadata": {"response_id": response_id}} + + def _previous_response_id(self) -> Optional[str]: + """Most recent stored Responses API id from chat history, if any.""" + for turn in reversed(self.chat_history or []): + if not isinstance(turn, dict): + continue + meta = turn.get("metadata") + if isinstance(meta, dict) and meta.get("response_id"): + return meta["response_id"] + return None @abstractmethod def _gen_inner( @@ -280,6 +300,7 @@ class BaseAgent(ABC): yield {"sources": self.retrieved_docs} yield {"tool_calls": self._get_truncated_tool_calls()} + yield from self._emit_responses_metadata() # ---- Tool delegation (thin wrappers around ToolExecutor) ---- @@ -558,6 +579,14 @@ class BaseAgent(ABC): gen_kwargs["response_format"] = structured_format elif self.llm_name == "google": gen_kwargs["response_schema"] = structured_format + if ( + settings.OPENAI_RESPONSES_STORE + and hasattr(self.llm, "_uses_responses_api") + and self.llm._uses_responses_api() + ): + previous_response_id = self._previous_response_id() + if previous_response_id: + gen_kwargs["previous_response_id"] = previous_response_id resp = self.llm.gen_stream(**gen_kwargs) if log_context: diff --git a/application/core/model_settings.py b/application/core/model_settings.py index 49843e57..a9c9372d 100644 --- a/application/core/model_settings.py +++ b/application/core/model_settings.py @@ -16,7 +16,6 @@ class ModelProvider(str, Enum): OPENAI = "openai" OPENAI_COMPATIBLE = "openai_compatible" OPENROUTER = "openrouter" - AZURE_OPENAI = "azure_openai" ANTHROPIC = "anthropic" GROQ = "groq" GOOGLE = "google" @@ -37,6 +36,14 @@ class ModelCapabilities: context_window: int = 128000 input_cost_per_token: Optional[float] = None output_cost_per_token: Optional[float] = None + # OpenAI reasoning-model effort hint (none/minimal/low/medium/high/xhigh; + # the accepted subset is model-dependent). Consumed only by the + # Responses API path in OpenAILLM; ignored by other providers. + reasoning_effort: Optional[str] = None + # Which OpenAI wire protocol the model speaks: "chat_completions" + # (the default) or "responses" (the /v1/responses endpoint). Set per + # model so only models that actually support the Responses API opt in. + api_flavor: str = "chat_completions" @dataclass diff --git a/application/core/model_yaml.py b/application/core/model_yaml.py index eb666c68..4a8323c4 100644 --- a/application/core/model_yaml.py +++ b/application/core/model_yaml.py @@ -31,6 +31,15 @@ logger = logging.getLogger(__name__) BUILTIN_MODELS_DIR = Path(__file__).parent / "models" DEFAULTS_FILENAME = "_defaults.yaml" +# Accepted reasoning_effort values across the OpenAI reasoning lineup. +# Validated at YAML load so a typo aborts boot rather than surfacing as a +# provider 400. The subset a given model actually accepts is model-dependent. +VALID_REASONING_EFFORTS = frozenset( + {"none", "minimal", "low", "medium", "high", "xhigh"} +) +# Accepted api_flavor values: which OpenAI wire protocol a model speaks. +VALID_API_FLAVORS = frozenset({"chat_completions", "responses"}) + class _DefaultsFile(BaseModel): """Schema for ``_defaults.yaml``. Currently just attachment aliases.""" @@ -56,6 +65,28 @@ class _CapabilityFields(BaseModel): context_window: Optional[int] = None input_cost_per_token: Optional[float] = None output_cost_per_token: Optional[float] = None + reasoning_effort: Optional[str] = None + api_flavor: Optional[str] = None + + @field_validator("reasoning_effort") + @classmethod + def _valid_reasoning_effort(cls, v: Optional[str]) -> Optional[str]: + if v is not None and v not in VALID_REASONING_EFFORTS: + valid = ", ".join(sorted(VALID_REASONING_EFFORTS)) + raise ValueError( + f"reasoning_effort must be one of [{valid}], got {v!r}" + ) + return v + + @field_validator("api_flavor") + @classmethod + def _valid_api_flavor(cls, v: Optional[str]) -> Optional[str]: + if v is not None and v not in VALID_API_FLAVORS: + valid = ", ".join(sorted(VALID_API_FLAVORS)) + raise ValueError( + f"api_flavor must be one of [{valid}], got {v!r}" + ) + return v class _ModelEntry(_CapabilityFields): @@ -205,6 +236,8 @@ def _build_model( context_window=pick("context_window", 128000), input_cost_per_token=pick("input_cost_per_token", None), output_cost_per_token=pick("output_cost_per_token", None), + reasoning_effort=pick("reasoning_effort", None), + api_flavor=pick("api_flavor", "chat_completions"), ) return AvailableModel( diff --git a/application/core/models/azure_openai.yaml b/application/core/models/azure_openai.yaml deleted file mode 100644 index b796a6cd..00000000 --- a/application/core/models/azure_openai.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# Azure OpenAI catalog. -# -# IMPORTANT: For Azure OpenAI, the `id` field is the **deployment name**, not -# a model name. Deployment names are arbitrary strings the operator chooses -# in Azure portal (or via ARM/Bicep/Terraform) when they create a deployment -# for a given underlying model + version. -# -# The IDs below are sensible defaults that mirror the underlying OpenAI -# model name (prefixed with `azure-`). Operators almost always need to -# override them via `MODELS_CONFIG_DIR` to match the deployment names that -# actually exist in their Azure resource. The `display_name`, capability -# flags, and `context_window` reflect the underlying OpenAI model. -provider: azure_openai - -defaults: - supports_tools: true - supports_structured_output: true - attachments: [image] - context_window: 400000 - -models: - - id: azure-gpt-5.5 - display_name: Azure OpenAI GPT-5.5 - description: Azure-hosted flagship frontier model for complex reasoning, coding, and agentic work with a 1M-token context window - context_window: 1050000 - - id: azure-gpt-5.4-mini - display_name: Azure OpenAI GPT-5.4 Mini - description: Azure-hosted cost-efficient GPT-5.4-class model for high-volume coding, computer use, and subagent workloads - - id: azure-gpt-5.4-nano - display_name: Azure OpenAI GPT-5.4 Nano - description: Azure-hosted cheapest GPT-5.4-class model, optimized for simple high-volume tasks where speed and cost matter most diff --git a/application/core/models/openai.yaml b/application/core/models/openai.yaml index 1f84e157..e0f209c5 100644 --- a/application/core/models/openai.yaml +++ b/application/core/models/openai.yaml @@ -10,6 +10,8 @@ models: display_name: GPT-5.5 description: Flagship frontier model for complex reasoning, coding, and agentic work with a 1M-token context window context_window: 1050000 + api_flavor: responses + reasoning_effort: medium - id: gpt-5.4-mini display_name: GPT-5.4 Mini description: Cost-efficient GPT-5.4-class model for high-volume coding, computer use, and subagent workloads diff --git a/application/core/settings.py b/application/core/settings.py index 0dee0db6..a5b88cf9 100644 --- a/application/core/settings.py +++ b/application/core/settings.py @@ -189,6 +189,13 @@ class Settings(BaseSettings): # Tool pre-fetch settings ENABLE_TOOL_PREFETCH: bool = True + # When True, OpenAI Responses API calls are persisted server-side + # (store=true) so a previous_response_id can chain turns. When False + # (the default) Responses calls are stateless (store=false) and any + # reasoning is carried across the in-turn tool loop via encrypted + # reasoning items instead. + OPENAI_RESPONSES_STORE: bool = False + # Config-free tools on by default in agentless chats. ``scheduler`` is # dual-registered (also in ``BUILTIN_AGENT_TOOLS``) so the same synthetic id # resolves whether reached via defaults or the agent picker. diff --git a/application/llm/openai.py b/application/llm/openai.py index 69228d01..dc17d5c8 100644 --- a/application/llm/openai.py +++ b/application/llm/openai.py @@ -60,6 +60,53 @@ def _truncate_base64_for_logging(messages): return truncated +class _RespFunction: + """Minimal stand-in for an OpenAI tool-call ``function`` object.""" + + def __init__(self, name, arguments): + self.name = name + self.arguments = arguments + + +class _RespToolCall: + """Chat-Completions-shaped tool call synthesized from a Responses + ``function_call`` item, so the existing OpenAI handler and the streaming + tool-call accumulator consume it unchanged.""" + + def __init__(self, id, index, name, arguments): + self.id = id + self.index = index + self.type = "function" + self.function = _RespFunction(name, arguments) + + +class _RespDelta: + """Stand-in for a streaming chat ``choice.delta``.""" + + def __init__(self, content=None, tool_calls=None): + self.content = content + self.tool_calls = tool_calls + + +class _RespMessage: + """Stand-in for a non-streaming chat ``choice.message``.""" + + def __init__(self, content=None, tool_calls=None): + self.content = content + self.tool_calls = tool_calls + + +class _RespChoice: + """Stand-in for ``response.choices[0]`` (non-streaming) or a streaming + chunk's choice. ``parse_response`` reads ``.message`` or ``.delta`` plus + ``.finish_reason``.""" + + def __init__(self, finish_reason, delta=None, message=None): + self.delta = delta + self.message = message + self.finish_reason = finish_reason + + class OpenAILLM(BaseLLM): provider_name = "openai" @@ -102,6 +149,13 @@ class OpenAILLM(BaseLLM): api_key=self.api_key, base_url=effective_base_url ) self.storage = StorageCreator.get_storage() + # Per-instance state for the Responses API path. ``_reasoning_for_calls`` + # maps a function-call id to the reasoning items that preceded it, so + # the model's chain-of-thought survives the in-turn tool round-trip. + # ``_last_response_id`` is the most recent /v1/responses id, used to + # chain turns when OPENAI_RESPONSES_STORE is enabled. + self._reasoning_for_calls = {} + self._last_response_id = None def _clean_messages_openai(self, messages): cleaned_messages = [] @@ -283,6 +337,17 @@ class OpenAILLM(BaseLLM): if response_format and not self._supports_structured_output(): response_format = None + previous_response_id = kwargs.pop("previous_response_id", None) + if self._uses_responses_api(): + return self._responses_gen( + model, + messages, + tools=tools, + response_format=response_format, + previous_response_id=previous_response_id, + **kwargs, + ) + request_params = { "model": model, "messages": messages, @@ -326,6 +391,18 @@ class OpenAILLM(BaseLLM): if response_format and not self._supports_structured_output(): response_format = None + previous_response_id = kwargs.pop("previous_response_id", None) + if self._uses_responses_api(): + yield from self._responses_gen_stream( + model, + messages, + tools=tools, + response_format=response_format, + previous_response_id=previous_response_id, + **kwargs, + ) + return + request_params = { "model": model, "messages": messages, @@ -366,6 +443,401 @@ class OpenAILLM(BaseLLM): if hasattr(response, "close"): response.close() + # ---- Responses API (/v1/responses) ---- + + def _uses_responses_api(self): + """True when the model's registry capability opts it into the + ``/v1/responses`` endpoint.""" + return ( + self.capabilities is not None + and getattr(self.capabilities, "api_flavor", "chat_completions") + == "responses" + ) + + @staticmethod + def _responses_content_parts(role, content): + """Translate a cleaned chat ``content`` value into Responses content + parts (``input_text``/``output_text``/``input_image``/``input_file``).""" + text_type = "output_text" if role == "assistant" else "input_text" + parts = [] + if content is None: + return parts + if isinstance(content, str): + if content: + parts.append({"type": text_type, "text": content}) + return parts + if isinstance(content, list): + for item in content: + if not isinstance(item, dict): + continue + itype = item.get("type") + if itype == "text": + parts.append({"type": text_type, "text": item.get("text", "")}) + elif itype == "image_url": + url = (item.get("image_url") or {}).get("url") + if url: + parts.append({"type": "input_image", "image_url": url}) + elif itype == "file": + file_obj = item.get("file") or {} + file_part = {"type": "input_file"} + for key in ("file_id", "filename", "file_data"): + if file_obj.get(key): + file_part[key] = file_obj[key] + parts.append(file_part) + return parts + + def _to_responses_input(self, messages): + """Translate cleaned Chat-Completions messages into a Responses + ``input`` item list. + + Reasoning items captured during the in-turn tool loop are re-injected + ahead of the function calls they belong to (deduped by id) so the + model keeps its chain-of-thought across the round-trip. + """ + input_items = [] + emitted_reasoning = set() + for message in messages: + role = message.get("role") + tool_calls = message.get("tool_calls") + if tool_calls and role == "assistant": + for tc in tool_calls: + call_id = tc.get("id", "") + for item in self._reasoning_for_calls.get(call_id, []): + item_id = item.get("id") + if item_id and item_id in emitted_reasoning: + continue + if item_id: + emitted_reasoning.add(item_id) + input_items.append(item) + func = tc.get("function", {}) + input_items.append({ + "type": "function_call", + "call_id": call_id, + "name": func.get("name", ""), + "arguments": func.get("arguments", "") or "{}", + }) + continue + tool_call_id = message.get("tool_call_id") + if role == "tool" and tool_call_id is not None: + tool_content = message.get("content") + input_items.append({ + "type": "function_call_output", + "call_id": tool_call_id, + "output": ( + tool_content + if isinstance(tool_content, str) + else json.dumps(tool_content) + ), + }) + continue + parts = self._responses_content_parts(role, message.get("content")) + if parts: + input_items.append({"role": role, "content": parts}) + return input_items + + @staticmethod + def _trim_for_previous_response(messages): + """When chaining via ``previous_response_id`` the server already holds + the earlier turns, so only system context plus everything after the + last completed assistant response needs to be sent again.""" + last_assistant = -1 + for i, message in enumerate(messages): + if message.get("role") == "assistant" and not message.get( + "tool_calls" + ): + last_assistant = i + if last_assistant < 0: + return messages + head = [ + m + for m in messages[: last_assistant + 1] + if m.get("role") == "system" + ] + return head + messages[last_assistant + 1:] + + @staticmethod + def _to_responses_tools(tools): + """Flatten Chat-Completions tool defs into Responses tool defs. + + ``strict`` is left False so schemas that were valid on Chat + Completions are not newly rejected by the stricter Responses default. + """ + converted = [] + for tool in tools or []: + if tool.get("type") == "function" and isinstance( + tool.get("function"), dict + ): + fn = tool["function"] + converted.append({ + "type": "function", + "name": fn.get("name", ""), + "description": fn.get("description", ""), + "parameters": fn.get("parameters", {}), + "strict": False, + }) + else: + converted.append(tool) + return converted + + @staticmethod + def _responses_text_format(response_format): + """Map a Chat-Completions ``response_format`` to a Responses + ``text.format`` object.""" + if not isinstance(response_format, dict): + return None + if response_format.get("type") == "json_schema": + js = response_format.get("json_schema", {}) + fmt = {"type": "json_schema", "name": js.get("name", "response")} + if "schema" in js: + fmt["schema"] = js["schema"] + if "strict" in js: + fmt["strict"] = js["strict"] + return fmt + if response_format.get("type") == "json_object": + return {"type": "json_object"} + return None + + def _build_responses_params( + self, + model, + input_items, + tools, + response_format, + previous_response_id, + stream, + kwargs, + ): + """Assemble the kwargs for ``client.responses.create``. Only known, + Responses-compatible keys are forwarded — unknown chat-only kwargs + are dropped so the API does not reject the request.""" + params = {"model": model, "input": input_items, "stream": stream} + + max_out = kwargs.pop("max_completion_tokens", None) + if max_out is None: + max_out = kwargs.pop("max_tokens", None) + if max_out is not None: + params["max_output_tokens"] = max_out + + effort = ( + getattr(self.capabilities, "reasoning_effort", None) + if self.capabilities is not None + else None + ) + if effort: + params["reasoning"] = {"effort": effort, "summary": "auto"} + + if response_format: + fmt = self._responses_text_format(response_format) + if fmt: + params["text"] = {"format": fmt} + + if tools: + params["tools"] = self._to_responses_tools(tools) + + store = bool(settings.OPENAI_RESPONSES_STORE) + params["store"] = store + if store: + if previous_response_id: + params["previous_response_id"] = previous_response_id + else: + # Stateless: ask the API to return encrypted reasoning so it can + # be replayed across the in-turn tool loop without server-side + # retention of the conversation. + params["include"] = ["reasoning.encrypted_content"] + return params + + @staticmethod + def _reasoning_item_to_dict(item): + """Serialize a Responses ``reasoning`` output item into the input + shape needed to feed it back on the next call.""" + result = {"type": "reasoning", "id": getattr(item, "id", None)} + encrypted = getattr(item, "encrypted_content", None) + if encrypted is not None: + result["encrypted_content"] = encrypted + summary = getattr(item, "summary", None) or [] + serialized = [] + for part in summary: + if isinstance(part, dict): + serialized.append(part) + else: + serialized.append({ + "type": getattr(part, "type", "summary_text"), + "text": getattr(part, "text", ""), + }) + result["summary"] = serialized + return result + + def _record_responses_metadata(self, response): + rid = getattr(response, "id", None) + if rid: + self._last_response_id = rid + + def _remember_reasoning(self, tool_calls, reasoning_items): + """Key captured reasoning items by each function-call id for replay + on the next in-turn request.""" + if not reasoning_items: + return + for tc in tool_calls: + self._reasoning_for_calls[tc.id] = reasoning_items + + def _parse_responses_output(self, response): + """Walk a non-streaming Responses ``output`` array into + ``(content, tool_calls, reasoning_items)``.""" + content_parts = [] + tool_calls = [] + reasoning_items = [] + for item in getattr(response, "output", None) or []: + itype = getattr(item, "type", None) + if itype == "reasoning": + reasoning_items.append(self._reasoning_item_to_dict(item)) + elif itype == "message": + for part in getattr(item, "content", None) or []: + if getattr(part, "type", None) == "output_text": + content_parts.append(getattr(part, "text", "") or "") + elif itype == "function_call": + tool_calls.append(_RespToolCall( + id=getattr(item, "call_id", "") or getattr(item, "id", ""), + index=len(tool_calls), + name=getattr(item, "name", "") or "", + arguments=getattr(item, "arguments", "") or "", + )) + return "".join(content_parts), tool_calls, reasoning_items + + def _responses_gen( + self, + model, + messages, + tools=None, + response_format=None, + previous_response_id=None, + **kwargs, + ): + if previous_response_id and settings.OPENAI_RESPONSES_STORE: + messages = self._trim_for_previous_response(messages) + input_items = self._to_responses_input(messages) + params = self._build_responses_params( + model, + input_items, + tools, + response_format, + previous_response_id, + stream=False, + kwargs=kwargs, + ) + response = self.client.responses.create(**params) + logging.info(f"OpenAI responses output: {getattr(response, 'output', None)}") + self._record_responses_metadata(response) + content, tool_calls, reasoning_items = self._parse_responses_output( + response + ) + if tools: + self._remember_reasoning(tool_calls, reasoning_items) + message = _RespMessage( + content=content or None, tool_calls=tool_calls or None + ) + return _RespChoice( + finish_reason="tool_calls" if tool_calls else "stop", + message=message, + ) + return content or "" + + def _responses_gen_stream( + self, + model, + messages, + tools=None, + response_format=None, + previous_response_id=None, + **kwargs, + ): + if previous_response_id and settings.OPENAI_RESPONSES_STORE: + messages = self._trim_for_previous_response(messages) + input_items = self._to_responses_input(messages) + params = self._build_responses_params( + model, + input_items, + tools, + response_format, + previous_response_id, + stream=True, + kwargs=kwargs, + ) + response = self.client.responses.create(**params) + + func_calls = {} + reasoning_items = [] + try: + for event in response: + etype = getattr(event, "type", "") + if etype == "response.output_text.delta": + delta = getattr(event, "delta", "") + if delta: + yield delta + elif etype == "response.reasoning_summary_text.delta": + delta = getattr(event, "delta", "") + if delta: + yield {"type": "thought", "thought": delta} + elif etype == "response.output_item.added": + item = getattr(event, "item", None) + if getattr(item, "type", None) == "function_call": + index = getattr(event, "output_index", len(func_calls)) + func_calls[index] = { + "call_id": ( + getattr(item, "call_id", "") + or getattr(item, "id", "") + ), + "name": getattr(item, "name", "") or "", + "arguments": "", + } + elif etype == "response.function_call_arguments.delta": + index = getattr(event, "output_index", None) + if index in func_calls: + func_calls[index]["arguments"] += ( + getattr(event, "delta", "") or "" + ) + elif etype == "response.function_call_arguments.done": + index = getattr(event, "output_index", None) + if index in func_calls: + done_args = getattr(event, "arguments", None) + if done_args is not None: + func_calls[index]["arguments"] = done_args + elif etype == "response.output_item.done": + item = getattr(event, "item", None) + if getattr(item, "type", None) == "reasoning": + reasoning_items.append( + self._reasoning_item_to_dict(item) + ) + elif etype == "response.completed": + self._record_responses_metadata( + getattr(event, "response", None) + ) + if func_calls: + tool_calls = [] + for position, index in enumerate(sorted(func_calls)): + entry = func_calls[index] + tool_calls.append(_RespToolCall( + id=entry["call_id"], + index=position, + name=entry["name"], + arguments=entry["arguments"], + )) + self._remember_reasoning(tool_calls, reasoning_items) + yield _RespChoice( + finish_reason="tool_calls", + delta=_RespDelta(tool_calls=tool_calls), + ) + elif etype in ("response.failed", "error"): + resp = getattr(event, "response", None) + err = ( + getattr(resp, "error", None) + or getattr(event, "message", None) + or "Responses stream error" + ) + raise RuntimeError(f"Responses API stream error: {err}") + finally: + if hasattr(response, "close"): + response.close() + def _supports_tools(self): # When the LLM was constructed via LLMCreator with a registered # AvailableModel, ``self.capabilities`` is the per-model record. @@ -620,20 +1092,3 @@ class OpenAILLM(BaseLLM): except Exception as e: logging.error(f"Error uploading file to OpenAI: {e}", exc_info=True) raise - - -class AzureOpenAILLM(OpenAILLM): - - def __init__(self, api_key, user_api_key, *args, **kwargs): - - super().__init__(api_key) - self.api_base = (settings.OPENAI_API_BASE,) - self.api_version = (settings.OPENAI_API_VERSION,) - self.deployment_name = (settings.AZURE_DEPLOYMENT_NAME,) - from openai import AzureOpenAI - - self.client = AzureOpenAI( - api_key=api_key, - api_version=settings.OPENAI_API_VERSION, - azure_endpoint=settings.OPENAI_API_BASE, - ) diff --git a/application/llm/providers/__init__.py b/application/llm/providers/__init__.py index 8b9315cb..c712c89e 100644 --- a/application/llm/providers/__init__.py +++ b/application/llm/providers/__init__.py @@ -11,7 +11,6 @@ from __future__ import annotations from typing import Dict, List from application.llm.providers.anthropic import AnthropicProvider -from application.llm.providers.azure_openai import AzureOpenAIProvider from application.llm.providers.base import Provider from application.llm.providers.docsgpt import DocsGPTProvider from application.llm.providers.google import GoogleProvider @@ -34,7 +33,6 @@ ALL_PROVIDERS: List[Provider] = [ DocsGPTProvider(), OpenAIProvider(), OpenAICompatibleProvider(), - AzureOpenAIProvider(), AnthropicProvider(), GoogleProvider(), GroqProvider(), diff --git a/application/llm/providers/azure_openai.py b/application/llm/providers/azure_openai.py deleted file mode 100644 index 87658c7a..00000000 --- a/application/llm/providers/azure_openai.py +++ /dev/null @@ -1,30 +0,0 @@ -from __future__ import annotations - -from typing import Optional - -from application.llm.openai import AzureOpenAILLM -from application.llm.providers.base import Provider - - -class AzureOpenAIProvider(Provider): - name = "azure_openai" - llm_class = AzureOpenAILLM - - def get_api_key(self, settings) -> Optional[str]: - # Azure historically uses the generic API_KEY field. - return settings.API_KEY - - def is_enabled(self, settings) -> bool: - if settings.OPENAI_API_BASE: - return True - return settings.LLM_PROVIDER == self.name and bool(settings.API_KEY) - - def filter_yaml_models(self, settings, models): - # Mirrors _add_azure_openai_models: when LLM_PROVIDER==azure_openai - # and LLM_NAME matches a known model, narrow to that one model. - # Otherwise load the entire catalog. - if settings.LLM_PROVIDER == self.name and settings.LLM_NAME: - named = [m for m in models if m.id == settings.LLM_NAME] - if named: - return named - return models diff --git a/tests/core/test_model_registry_yaml.py b/tests/core/test_model_registry_yaml.py index fdf6f6ae..df1105c3 100644 --- a/tests/core/test_model_registry_yaml.py +++ b/tests/core/test_model_registry_yaml.py @@ -60,11 +60,6 @@ EXPECTED_IDS = { "deepseek-v4-flash", "deepseek-v4-pro", }, - "azure_openai": { - "azure-gpt-5.5", - "azure-gpt-5.4-mini", - "azure-gpt-5.4-nano", - }, "docsgpt": {"docsgpt-local"}, "huggingface": {"huggingface-local"}, } @@ -241,20 +236,6 @@ class TestRegistryPermutations: ids = {m.id for m in reg.get_all_models()} assert ids == EXPECTED_IDS["docsgpt"] - def test_azure_via_provider(self): - s = _make_settings(LLM_PROVIDER="azure_openai", API_KEY="key") - with patch("application.core.settings.settings", s): - reg = ModelRegistry() - ids = {m.id for m in reg.get_all_models()} - assert "azure-gpt-5.5" in ids - - def test_azure_via_api_base(self): - s = _make_settings(OPENAI_API_BASE="https://x.openai.azure.com") - with patch("application.core.settings.settings", s): - reg = ModelRegistry() - ids = {m.id for m in reg.get_all_models()} - assert "azure-gpt-5.5" in ids - def test_everything_set(self, monkeypatch): monkeypatch.setenv("DEEPSEEK_API_KEY", "x") s = _make_settings( diff --git a/tests/core/test_model_settings.py b/tests/core/test_model_settings.py index d4e1cec4..f00de9c4 100644 --- a/tests/core/test_model_settings.py +++ b/tests/core/test_model_settings.py @@ -37,7 +37,6 @@ class TestModelProvider: assert ModelProvider.SAGEMAKER == "sagemaker" assert ModelProvider.PREMAI == "premai" assert ModelProvider.LLAMA_CPP == "llama.cpp" - assert ModelProvider.AZURE_OPENAI == "azure_openai" class TestModelCapabilities: diff --git a/tests/core/test_model_utils.py b/tests/core/test_model_utils.py index 7aefeeb5..4d6d65b7 100644 --- a/tests/core/test_model_utils.py +++ b/tests/core/test_model_utils.py @@ -156,16 +156,6 @@ class TestGetApiKeyForProvider: assert get_api_key_for_provider("unknown_provider") == "sk-fallback" - @pytest.mark.unit - def test_azure_openai_key(self): - with patch("application.core.settings.settings") as mock_settings: - mock_settings.API_KEY = "sk-azure" - - from application.core.model_utils import get_api_key_for_provider - - assert get_api_key_for_provider("azure_openai") == "sk-azure" - - # ── get_all_available_models ───────────────────────────────────────────────── diff --git a/tests/llm/test_openai.py b/tests/llm/test_openai.py index 973dbe2f..486c5ebf 100644 --- a/tests/llm/test_openai.py +++ b/tests/llm/test_openai.py @@ -8,7 +8,6 @@ Extends coverage beyond test_openai_llm.py: - _raw_gen with tools and response_format - _raw_gen_stream tool_calls yielding - prepare_structured_output_format nested schemas - - AzureOpenAILLM constructor - _supports_tools / _supports_structured_output - get_supported_attachment_types - prepare_messages_with_attachments edge cases @@ -751,44 +750,6 @@ class TestGetBase64Image: llm._get_base64_image({"path": "/nonexistent"}) -# AzureOpenAILLM - - -@pytest.mark.unit -class TestAzureOpenAILLM: - - def test_constructor(self, monkeypatch): - monkeypatch.setattr( - "application.llm.openai.settings", - types.SimpleNamespace( - OPENAI_API_KEY="k", - API_KEY="k", - OPENAI_BASE_URL="", - OPENAI_API_BASE="https://my.azure.endpoint", - OPENAI_API_VERSION="2024-02-01", - AZURE_DEPLOYMENT_NAME="my-deployment", - ), - ) - monkeypatch.setattr( - "application.llm.openai.StorageCreator", - types.SimpleNamespace(get_storage=lambda: None), - ) - from unittest.mock import MagicMock - - monkeypatch.setattr("application.llm.openai.OpenAI", MagicMock()) - mock_azure = MagicMock() - monkeypatch.setattr("openai.AzureOpenAI", mock_azure, raising=False) - - # We need to reimport to get fresh class with mocked module - import importlib - import application.llm.openai as oai_mod - - importlib.reload(oai_mod) - - # Just verify the class exists and inherits from OpenAILLM - assert issubclass(oai_mod.AzureOpenAILLM, oai_mod.OpenAILLM) - - # _truncate_base64_for_logging — additional edges diff --git a/tests/llm/test_openai_responses.py b/tests/llm/test_openai_responses.py new file mode 100644 index 00000000..a7505a49 --- /dev/null +++ b/tests/llm/test_openai_responses.py @@ -0,0 +1,366 @@ +"""Unit tests for the OpenAI Responses API path in application/llm/openai.py. + +Covers the api_flavor gating, Chat-Completions -> Responses request +translation, tool/structured-output mapping, reasoning-item carryover, the +streaming-event normalization into the existing handler contract, and the +previous_response_id trimming used for cross-turn chaining. +""" + +import types +from unittest.mock import MagicMock + +import pytest + +from application.core.model_settings import ModelCapabilities + + +def _make_llm(monkeypatch, capabilities=None, store_responses=False): + monkeypatch.setattr("application.llm.openai.OpenAI", MagicMock()) + monkeypatch.setattr( + "application.llm.openai.StorageCreator", + types.SimpleNamespace(get_storage=lambda: None), + ) + monkeypatch.setattr( + "application.llm.openai.settings", + types.SimpleNamespace( + OPENAI_API_KEY="k", + API_KEY="k", + OPENAI_BASE_URL="", + AZURE_DEPLOYMENT_NAME="dep", + OPENAI_RESPONSES_STORE=store_responses, + ), + ) + from application.llm.openai import OpenAILLM + + llm = OpenAILLM(api_key="k") + llm.capabilities = capabilities + return llm + + +def _ns(**kw): + return types.SimpleNamespace(**kw) + + +def _responses_caps(reasoning_effort=None): + return ModelCapabilities( + supports_tools=True, + supports_structured_output=True, + api_flavor="responses", + reasoning_effort=reasoning_effort, + ) + + +# ── api_flavor gating ──────────────────────────────────────────────────────── + + +@pytest.mark.unit +def test_uses_responses_api_true(monkeypatch): + llm = _make_llm(monkeypatch, _responses_caps()) + assert llm._uses_responses_api() is True + + +@pytest.mark.unit +def test_uses_responses_api_false_for_chat(monkeypatch): + caps = ModelCapabilities(api_flavor="chat_completions") + assert _make_llm(monkeypatch, caps)._uses_responses_api() is False + + +@pytest.mark.unit +def test_uses_responses_api_false_without_caps(monkeypatch): + assert _make_llm(monkeypatch, None)._uses_responses_api() is False + + +# ── message translation ────────────────────────────────────────────────────── + + +@pytest.mark.unit +def test_to_responses_input_tool_roundtrip(monkeypatch): + llm = _make_llm(monkeypatch, _responses_caps()) + messages = [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": None, + "tool_calls": [{ + "id": "call_1", + "type": "function", + "function": {"name": "search", "arguments": '{"q":"x"}'}, + }], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "result"}, + {"role": "assistant", "content": "final"}, + ] + items = llm._to_responses_input(messages) + assert items == [ + {"role": "system", "content": [{"type": "input_text", "text": "sys"}]}, + {"role": "user", "content": [{"type": "input_text", "text": "hi"}]}, + { + "type": "function_call", + "call_id": "call_1", + "name": "search", + "arguments": '{"q":"x"}', + }, + {"type": "function_call_output", "call_id": "call_1", "output": "result"}, + {"role": "assistant", "content": [{"type": "output_text", "text": "final"}]}, + ] + + +@pytest.mark.unit +def test_to_responses_input_reinjects_reasoning(monkeypatch): + llm = _make_llm(monkeypatch, _responses_caps()) + reasoning_item = { + "type": "reasoning", "id": "rs_1", + "encrypted_content": "enc", "summary": [], + } + llm._reasoning_for_calls = {"call_1": [reasoning_item]} + messages = [{ + "role": "assistant", + "content": None, + "tool_calls": [{ + "id": "call_1", + "type": "function", + "function": {"name": "t", "arguments": "{}"}, + }], + }] + items = llm._to_responses_input(messages) + # Reasoning item is emitted immediately before its function call. + assert items[0] == reasoning_item + assert items[1]["type"] == "function_call" + assert items[1]["call_id"] == "call_1" + + +@pytest.mark.unit +def test_to_responses_input_multimodal_image(monkeypatch): + llm = _make_llm(monkeypatch, _responses_caps()) + messages = [{ + "role": "user", + "content": [ + {"type": "text", "text": "look"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,xx"}}, + ], + }] + items = llm._to_responses_input(messages) + assert items == [{ + "role": "user", + "content": [ + {"type": "input_text", "text": "look"}, + {"type": "input_image", "image_url": "data:image/png;base64,xx"}, + ], + }] + + +@pytest.mark.unit +def test_to_responses_tools_flatten(monkeypatch): + llm = _make_llm(monkeypatch, _responses_caps()) + tools = [{ + "type": "function", + "function": { + "name": "search", + "description": "Search", + "parameters": {"type": "object", "properties": {}}, + }, + }] + assert llm._to_responses_tools(tools) == [{ + "type": "function", + "name": "search", + "description": "Search", + "parameters": {"type": "object", "properties": {}}, + "strict": False, + }] + + +@pytest.mark.unit +def test_responses_text_format_json_schema(monkeypatch): + llm = _make_llm(monkeypatch, _responses_caps()) + rf = { + "type": "json_schema", + "json_schema": {"name": "out", "schema": {"type": "object"}, "strict": True}, + } + assert llm._responses_text_format(rf) == { + "type": "json_schema", "name": "out", + "schema": {"type": "object"}, "strict": True, + } + + +@pytest.mark.unit +def test_trim_for_previous_response(monkeypatch): + llm = _make_llm(monkeypatch, _responses_caps()) + messages = [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "old q"}, + {"role": "assistant", "content": "old a"}, + {"role": "user", "content": "new q"}, + ] + trimmed = llm._trim_for_previous_response(messages) + # System stays; everything up to and including the last assistant text + # is dropped (the server already holds it), leaving the new user turn. + assert trimmed == [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "new q"}, + ] + + +# ── request params ─────────────────────────────────────────────────────────── + + +@pytest.mark.unit +def test_build_responses_params_stateless(monkeypatch): + llm = _make_llm(monkeypatch, _responses_caps(reasoning_effort="high")) + params = llm._build_responses_params( + "gpt-5.5", [{"role": "user", "content": []}], tools=None, + response_format=None, previous_response_id=None, stream=True, + kwargs={"max_completion_tokens": 256}, + ) + assert params["model"] == "gpt-5.5" + assert params["stream"] is True + assert params["max_output_tokens"] == 256 + assert params["reasoning"] == {"effort": "high", "summary": "auto"} + assert params["store"] is False + assert params["include"] == ["reasoning.encrypted_content"] + assert "previous_response_id" not in params + + +@pytest.mark.unit +def test_build_responses_params_store_with_previous_id(monkeypatch): + llm = _make_llm(monkeypatch, _responses_caps(), store_responses=True) + params = llm._build_responses_params( + "gpt-5.5", [], tools=None, response_format=None, + previous_response_id="resp_abc", stream=False, kwargs={}, + ) + assert params["store"] is True + assert params["previous_response_id"] == "resp_abc" + assert "include" not in params + + +# ── streaming normalization into the existing handler contract ─────────────── + + +@pytest.mark.unit +def test_responses_gen_stream_text_and_tools(monkeypatch): + from application.llm.handlers.openai import OpenAILLMHandler + + llm = _make_llm(monkeypatch, _responses_caps()) + events = [ + _ns(type="response.output_text.delta", delta="Hel"), + _ns(type="response.output_text.delta", delta="lo"), + _ns(type="response.reasoning_summary_text.delta", delta="thinking"), + _ns( + type="response.output_item.added", + output_index=0, + item=_ns(type="function_call", call_id="call_1", name="search", id="fc_1"), + ), + _ns(type="response.function_call_arguments.delta", output_index=0, delta='{"q":'), + _ns( + type="response.function_call_arguments.done", + output_index=0, + arguments='{"q":"hi"}', + ), + _ns( + type="response.output_item.done", + item=_ns(type="reasoning", id="rs_1", encrypted_content="enc", summary=[]), + ), + _ns(type="response.completed", response=_ns(id="resp_1")), + ] + llm.client.responses.create = MagicMock(return_value=events) + + out = list(llm._responses_gen_stream("gpt-5.5", [{"role": "user", "content": "hi"}], tools=[{"type": "function", "function": {"name": "search", "parameters": {}}}])) + + assert "Hel" in out and "lo" in out + assert {"type": "thought", "thought": "thinking"} in out + choice = out[-1] + parsed = OpenAILLMHandler().parse_response(choice) + assert parsed.finish_reason == "tool_calls" + assert len(parsed.tool_calls) == 1 + tc = parsed.tool_calls[0] + assert tc.id == "call_1" + assert tc.name == "search" + assert tc.arguments == '{"q":"hi"}' + # Reasoning captured for in-turn carryover, last response id recorded. + assert llm._reasoning_for_calls["call_1"][0]["encrypted_content"] == "enc" + assert llm._last_response_id == "resp_1" + + +@pytest.mark.unit +def test_responses_gen_stream_text_only(monkeypatch): + llm = _make_llm(monkeypatch, _responses_caps()) + events = [ + _ns(type="response.output_text.delta", delta="Answer"), + _ns(type="response.completed", response=_ns(id="resp_2")), + ] + llm.client.responses.create = MagicMock(return_value=events) + out = list(llm._responses_gen_stream("gpt-5.5", [{"role": "user", "content": "hi"}], tools=None)) + assert out == ["Answer"] + assert llm._last_response_id == "resp_2" + + +@pytest.mark.unit +def test_responses_gen_nonstream_tools(monkeypatch): + from application.llm.handlers.openai import OpenAILLMHandler + + llm = _make_llm(monkeypatch, _responses_caps()) + response = _ns( + id="resp_3", + output=[ + _ns(type="reasoning", id="rs", encrypted_content="e", summary=[]), + _ns(type="message", content=[_ns(type="output_text", text="Answer")]), + _ns(type="function_call", call_id="c1", name="t", arguments="{}", id="fc"), + ], + ) + llm.client.responses.create = MagicMock(return_value=response) + choice = llm._responses_gen("gpt-5.5", [{"role": "user", "content": "hi"}], tools=[{"type": "function", "function": {"name": "t", "parameters": {}}}]) + parsed = OpenAILLMHandler().parse_response(choice) + assert parsed.finish_reason == "tool_calls" + assert parsed.tool_calls[0].id == "c1" + assert llm._reasoning_for_calls["c1"][0]["encrypted_content"] == "e" + + +@pytest.mark.unit +def test_responses_gen_nonstream_text(monkeypatch): + llm = _make_llm(monkeypatch, _responses_caps()) + response = _ns( + id="resp_4", + output=[_ns(type="message", content=[_ns(type="output_text", text="Hi there")])], + ) + llm.client.responses.create = MagicMock(return_value=response) + result = llm._responses_gen("gpt-5.5", [{"role": "user", "content": "hi"}], tools=None) + assert result == "Hi there" + + +# ── capability plumbing / yaml ─────────────────────────────────────────────── + + +@pytest.mark.unit +def test_capability_field_rejects_bad_api_flavor(): + from application.core.model_yaml import _CapabilityFields + + with pytest.raises(ValueError): + _CapabilityFields(api_flavor="grpc") + + +@pytest.mark.unit +def test_capability_field_rejects_bad_reasoning_effort(): + from application.core.model_yaml import _CapabilityFields + + with pytest.raises(ValueError): + _CapabilityFields(reasoning_effort="extreme") + + +@pytest.mark.unit +def test_builtin_gpt55_opts_into_responses(): + from application.core.model_yaml import BUILTIN_MODELS_DIR, load_model_yamls + + catalogs = load_model_yamls([BUILTIN_MODELS_DIR]) + models = {m.id: m for c in catalogs for m in c.models} + gpt = models["gpt-5.5"] + assert gpt.capabilities.api_flavor == "responses" + assert gpt.capabilities.reasoning_effort == "medium" + + +@pytest.mark.unit +def test_builtin_default_models_stay_chat_completions(): + from application.core.model_yaml import BUILTIN_MODELS_DIR, load_model_yamls + + catalogs = load_model_yamls([BUILTIN_MODELS_DIR]) + models = {m.id: m for c in catalogs for m in c.models} + assert models["gpt-5.4-mini"].capabilities.api_flavor == "chat_completions" From 27f9ddc8e0683189aded279f18163d2d6119d88b Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 3 Jun 2026 14:56:28 +0100 Subject: [PATCH 2/3] =?UTF-8?q?fix:=20address=20PR=20review=20=E2=80=94=20?= =?UTF-8?q?Responses=20input=20encoding=20+=20finish=20Azure=20removal?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replay prior-turn assistant text as input_text, not output_text: a Responses easy-input message only accepts input_* content parts, so the old shape would 400 on the second turn of every conversation (default store=false resends prior turns inline). - Always request include=["reasoning.encrypted_content"] so in-turn reasoning carryover works whether or not the response is also stored server-side (OPENAI_RESPONSES_STORE). - Add detail="auto" to input_image parts. - Remove the now-dead azure_openai option from setup.sh / setup.ps1 and the docs, completing the AzureOpenAILLM removal. - Tests: correct the input_text / image-detail / include assertions and add parallel tool-call and stream-error coverage. --- application/llm/openai.py | 32 ++++++++----- docs/content/Deploying/DocsGPT-Settings.mdx | 3 +- docs/content/Models/cloud-providers.mdx | 1 - setup.ps1 | 30 ++---------- setup.sh | 28 ++--------- tests/llm/test_openai_responses.py | 53 +++++++++++++++++++-- 6 files changed, 80 insertions(+), 67 deletions(-) diff --git a/application/llm/openai.py b/application/llm/openai.py index 7942c220..ddbe5e3d 100644 --- a/application/llm/openai.py +++ b/application/llm/openai.py @@ -461,8 +461,15 @@ class OpenAILLM(BaseLLM): @staticmethod def _responses_content_parts(role, content): """Translate a cleaned chat ``content`` value into Responses content - parts (``input_text``/``output_text``/``input_image``/``input_file``).""" - text_type = "output_text" if role == "assistant" else "input_text" + parts (``input_text``/``input_image``/``input_file``). + + Assistant text from prior turns is replayed as ``input_text`` too: a + Responses easy-input message only accepts the input_* content types + regardless of role (``output_text`` is reserved for full + output-message items, which carry id/status/annotations). + """ + _ = role + text_type = "input_text" parts = [] if content is None: return parts @@ -480,7 +487,11 @@ class OpenAILLM(BaseLLM): elif itype == "image_url": url = (item.get("image_url") or {}).get("url") if url: - parts.append({"type": "input_image", "image_url": url}) + parts.append({ + "type": "input_image", + "image_url": url, + "detail": "auto", + }) elif itype == "file": file_obj = item.get("file") or {} file_part = {"type": "input_file"} @@ -640,14 +651,13 @@ class OpenAILLM(BaseLLM): store = bool(settings.OPENAI_RESPONSES_STORE) params["store"] = store - if store: - if previous_response_id: - params["previous_response_id"] = previous_response_id - else: - # Stateless: ask the API to return encrypted reasoning so it can - # be replayed across the in-turn tool loop without server-side - # retention of the conversation. - params["include"] = ["reasoning.encrypted_content"] + if store and previous_response_id: + params["previous_response_id"] = previous_response_id + # Always request encrypted reasoning content so reasoning items can be + # replayed by value across the in-turn tool loop — this keeps + # carryover working whether or not the response is also retained + # server-side (store=true). + params["include"] = ["reasoning.encrypted_content"] return params @staticmethod diff --git a/docs/content/Deploying/DocsGPT-Settings.mdx b/docs/content/Deploying/DocsGPT-Settings.mdx index 373e56cb..ea4227e9 100644 --- a/docs/content/Deploying/DocsGPT-Settings.mdx +++ b/docs/content/Deploying/DocsGPT-Settings.mdx @@ -48,7 +48,6 @@ Here are some of the most fundamental settings you'll likely want to configure: - `anthropic`: Use Anthropic's Claude models. - `groq`: Use Groq's models. - `huggingface`: Use HuggingFace Inference API. - - `azure_openai`: Use Azure OpenAI Service. - `openai` (when using local inference engines like Ollama, Llama.cpp, TGI, etc.): This signals DocsGPT to use an OpenAI-compatible API format, even if the actual LLM is running locally. - **`LLM_NAME`**: Specifies the specific model to use from the chosen LLM provider. The available models depend on the `LLM_PROVIDER` you've selected. @@ -103,7 +102,7 @@ In this case, even though you are using Ollama locally, `LLM_PROVIDER` is set to DocsGPT ships with a built-in catalog of models for the providers it supports out of the box (OpenAI, Anthropic, Google, Groq, OpenRouter, -Novita, Azure OpenAI, Hugging Face, DocsGPT). To add **your own +Novita, Hugging Face, DocsGPT). To add **your own models** without forking the repo — for example, a Mistral or Together account, a self-hosted vLLM endpoint, or any other OpenAI-compatible API — point `MODELS_CONFIG_DIR` at a directory of YAML files. diff --git a/docs/content/Models/cloud-providers.mdx b/docs/content/Models/cloud-providers.mdx index cc4018b3..23d0fd44 100644 --- a/docs/content/Models/cloud-providers.mdx +++ b/docs/content/Models/cloud-providers.mdx @@ -29,7 +29,6 @@ DocsGPT offers direct, streamlined support for the following cloud LLM providers | Anthropic (Claude) | `anthropic` | `claude-3-5-sonnet-20241022`| | Groq | `groq` | `llama-3.3-70b-versatile` | | HuggingFace Inference API | `huggingface` | `meta-llama/Llama-3.1-8B-Instruct` | -| Azure OpenAI | `azure_openai` | `azure-gpt-4` | | Prem AI | `premai` | (See Prem AI docs) | | AWS SageMaker | `sagemaker` | (See SageMaker docs) | | Novita AI | `novita` | (See Novita docs) | diff --git a/setup.ps1 b/setup.ps1 index 75a8011f..f3d43d31 100644 --- a/setup.ps1 +++ b/setup.ps1 @@ -266,11 +266,10 @@ function Prompt-CloudAPIProviderOptions { Write-ColorText "3) Anthropic (Claude)" -ForegroundColor "Yellow" Write-ColorText "4) Groq" -ForegroundColor "Yellow" Write-ColorText "5) HuggingFace Inference API" -ForegroundColor "Yellow" - Write-ColorText "6) Azure OpenAI" -ForegroundColor "Yellow" - Write-ColorText "7) Novita" -ForegroundColor "Yellow" + Write-ColorText "6) Novita" -ForegroundColor "Yellow" Write-ColorText "b) Back to Main Menu" -ForegroundColor "Yellow" Write-Host "" - $script:provider_choice = Read-Host "Choose option (1-7, or b)" + $script:provider_choice = Read-Host "Choose option (1-6, or b)" } # Function to prompt for Ollama CPU/GPU options @@ -974,20 +973,7 @@ function Connect-CloudAPIProvider { Get-APIKey break } - "6" { # Azure OpenAI - $script:provider_name = "Azure OpenAI" - $script:llm_name = "azure_openai" - $script:model_name = "gpt-4o" - Get-APIKey - Write-Host "" - Write-ColorText "Azure OpenAI requires additional configuration:" -ForegroundColor "White" -Bold - $script:azure_api_base = Read-Host "Enter Azure OpenAI API base URL (e.g. https://your-resource.openai.azure.com/)" - $script:azure_api_version = Read-Host "Enter Azure OpenAI API version (e.g. 2024-02-15-preview)" - $script:azure_deployment = Read-Host "Enter Azure deployment name for chat" - $script:azure_emb_deployment = Read-Host "Enter Azure deployment name for embeddings (leave empty to skip)" - break - } - "7" { # Novita + "6" { # Novita $script:provider_name = "Novita" $script:llm_provider = "novita" $script:model_name = "moonshotai/kimi-k2.5" @@ -998,7 +984,7 @@ function Connect-CloudAPIProvider { "B" { Clear-Host; return } default { Write-Host "" - Write-ColorText "Invalid choice. Please choose 1-7, or b." -ForegroundColor "Red" + Write-ColorText "Invalid choice. Please choose 1-6, or b." -ForegroundColor "Red" Start-Sleep -Seconds 1 } } @@ -1017,14 +1003,6 @@ function Connect-CloudAPIProvider { "LLM_NAME=$model_name" | Add-Content -Path $ENV_FILE -Encoding utf8 "VITE_API_STREAMING=true" | Add-Content -Path $ENV_FILE -Encoding utf8 - # Azure OpenAI additional settings - if ($llm_name -eq "azure_openai") { - if ($azure_api_base) { "OPENAI_API_BASE=$azure_api_base" | Add-Content -Path $ENV_FILE -Encoding utf8 } - if ($azure_api_version) { "OPENAI_API_VERSION=$azure_api_version" | Add-Content -Path $ENV_FILE -Encoding utf8 } - if ($azure_deployment) { "AZURE_DEPLOYMENT_NAME=$azure_deployment" | Add-Content -Path $ENV_FILE -Encoding utf8 } - if ($azure_emb_deployment) { "AZURE_EMBEDDINGS_DEPLOYMENT_NAME=$azure_emb_deployment" | Add-Content -Path $ENV_FILE -Encoding utf8 } - } - Write-ColorText ".env file configured for $provider_name." -ForegroundColor "Green" Prompt-AdvancedSettings diff --git a/setup.sh b/setup.sh index 7ae1d3cc..e98a6e0b 100755 --- a/setup.sh +++ b/setup.sh @@ -152,11 +152,10 @@ prompt_cloud_api_provider_options() { echo -e "${YELLOW}3) Anthropic (Claude)${NC}" echo -e "${YELLOW}4) Groq${NC}" echo -e "${YELLOW}5) HuggingFace Inference API${NC}" - echo -e "${YELLOW}6) Azure OpenAI${NC}" - echo -e "${YELLOW}7) Novita${NC}" + echo -e "${YELLOW}6) Novita${NC}" echo -e "${YELLOW}b) Back to Main Menu${NC}" echo - read -p "$(echo -e "${DEFAULT_FG}Choose option (1-7, or b): ${NC}")" provider_choice + read -p "$(echo -e "${DEFAULT_FG}Choose option (1-6, or b): ${NC}")" provider_choice } # Function to prompt for Ollama CPU/GPU options @@ -700,25 +699,14 @@ connect_cloud_api_provider() { model_name="meta-llama/Llama-3.1-8B-Instruct" get_api_key break ;; - 6) # Azure OpenAI - provider_name="Azure OpenAI" - llm_provider="azure_openai" - model_name="gpt-4o" - get_api_key - echo -e "\n${DEFAULT_FG}${BOLD}Azure OpenAI requires additional configuration:${NC}" - read -p "$(echo -e "${DEFAULT_FG}Enter Azure OpenAI API base URL (e.g. https://your-resource.openai.azure.com/): ${NC}")" azure_api_base - read -p "$(echo -e "${DEFAULT_FG}Enter Azure OpenAI API version (e.g. 2024-02-15-preview): ${NC}")" azure_api_version - read -p "$(echo -e "${DEFAULT_FG}Enter Azure deployment name for chat: ${NC}")" azure_deployment - read -p "$(echo -e "${DEFAULT_FG}Enter Azure deployment name for embeddings (leave empty to skip): ${NC}")" azure_emb_deployment - break ;; - 7) # Novita + 6) # Novita provider_name="Novita" llm_provider="novita" model_name="moonshotai/kimi-k2.5" get_api_key break ;; b|B) clear; return 1 ;; # Clear screen and Back to Main Menu - *) echo -e "\n${RED}Invalid choice. Please choose 1-7, or b.${NC}" ; sleep 1 ;; + *) echo -e "\n${RED}Invalid choice. Please choose 1-6, or b.${NC}" ; sleep 1 ;; esac done @@ -728,14 +716,6 @@ connect_cloud_api_provider() { echo "LLM_NAME=$model_name" >> "$ENV_FILE" echo "VITE_API_STREAMING=true" >> "$ENV_FILE" - # Azure OpenAI additional settings - if [ "$llm_provider" = "azure_openai" ]; then - [ -n "$azure_api_base" ] && echo "OPENAI_API_BASE=$azure_api_base" >> "$ENV_FILE" - [ -n "$azure_api_version" ] && echo "OPENAI_API_VERSION=$azure_api_version" >> "$ENV_FILE" - [ -n "$azure_deployment" ] && echo "AZURE_DEPLOYMENT_NAME=$azure_deployment" >> "$ENV_FILE" - [ -n "$azure_emb_deployment" ] && echo "AZURE_EMBEDDINGS_DEPLOYMENT_NAME=$azure_emb_deployment" >> "$ENV_FILE" - fi - echo -e "${GREEN}.env file configured for ${BOLD}${provider_name}${NC}${GREEN}.${NC}" prompt_advanced_settings diff --git a/tests/llm/test_openai_responses.py b/tests/llm/test_openai_responses.py index a7505a49..67a021d8 100644 --- a/tests/llm/test_openai_responses.py +++ b/tests/llm/test_openai_responses.py @@ -102,7 +102,7 @@ def test_to_responses_input_tool_roundtrip(monkeypatch): "arguments": '{"q":"x"}', }, {"type": "function_call_output", "call_id": "call_1", "output": "result"}, - {"role": "assistant", "content": [{"type": "output_text", "text": "final"}]}, + {"role": "assistant", "content": [{"type": "input_text", "text": "final"}]}, ] @@ -145,7 +145,11 @@ def test_to_responses_input_multimodal_image(monkeypatch): "role": "user", "content": [ {"type": "input_text", "text": "look"}, - {"type": "input_image", "image_url": "data:image/png;base64,xx"}, + { + "type": "input_image", + "image_url": "data:image/png;base64,xx", + "detail": "auto", + }, ], }] @@ -230,7 +234,9 @@ def test_build_responses_params_store_with_previous_id(monkeypatch): ) assert params["store"] is True assert params["previous_response_id"] == "resp_abc" - assert "include" not in params + # Encrypted reasoning is always requested so in-turn carryover works + # regardless of server-side retention. + assert params["include"] == ["reasoning.encrypted_content"] # ── streaming normalization into the existing handler contract ─────────────── @@ -294,6 +300,47 @@ def test_responses_gen_stream_text_only(monkeypatch): assert llm._last_response_id == "resp_2" +@pytest.mark.unit +def test_responses_gen_stream_parallel_tool_calls(monkeypatch): + from application.llm.handlers.openai import OpenAILLMHandler + + llm = _make_llm(monkeypatch, _responses_caps()) + events = [ + _ns( + type="response.output_item.added", output_index=0, + item=_ns(type="function_call", call_id="call_a", name="t1", id="fc_a"), + ), + _ns( + type="response.output_item.added", output_index=1, + item=_ns(type="function_call", call_id="call_b", name="t2", id="fc_b"), + ), + _ns(type="response.function_call_arguments.delta", output_index=0, delta='{"a":'), + _ns(type="response.function_call_arguments.done", output_index=0, arguments='{"a":1}'), + _ns(type="response.function_call_arguments.done", output_index=1, arguments='{"b":2}'), + _ns(type="response.completed", response=_ns(id="resp_p")), + ] + llm.client.responses.create = MagicMock(return_value=events) + out = list(llm._responses_gen_stream("gpt-5.5", [{"role": "user", "content": "hi"}], tools=[{"type": "function", "function": {"name": "t1", "parameters": {}}}])) + parsed = OpenAILLMHandler().parse_response(out[-1]) + assert parsed.finish_reason == "tool_calls" + assert [tc.id for tc in parsed.tool_calls] == ["call_a", "call_b"] + assert [tc.index for tc in parsed.tool_calls] == [0, 1] + assert parsed.tool_calls[0].arguments == '{"a":1}' + assert parsed.tool_calls[1].arguments == '{"b":2}' + + +@pytest.mark.unit +def test_responses_gen_stream_error_event_raises(monkeypatch): + llm = _make_llm(monkeypatch, _responses_caps()) + events = [ + _ns(type="response.output_text.delta", delta="partial"), + _ns(type="response.failed", response=_ns(error="boom")), + ] + llm.client.responses.create = MagicMock(return_value=events) + with pytest.raises(RuntimeError): + list(llm._responses_gen_stream("gpt-5.5", [{"role": "user", "content": "hi"}], tools=None)) + + @pytest.mark.unit def test_responses_gen_nonstream_tools(monkeypatch): from application.llm.handlers.openai import OpenAILLMHandler From 9a69e89096e9f5131afaf15369f6359a89599aec Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 3 Jun 2026 15:00:14 +0100 Subject: [PATCH 3/3] fix: assistant input content must be output_text, not input_text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live /v1/responses rejects input_text for the assistant role (input[N].content[0] -> "Supported values are: 'output_text' and 'refusal'"). Revert the previous review-driven change: encode assistant turns as output_text and user/system turns as input_text. The SDK type union suggested input_* only, but the API enforces output_text per-role — confirmed against the live endpoint. --- application/llm/openai.py | 13 +++++-------- tests/llm/test_openai_responses.py | 4 +++- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/application/llm/openai.py b/application/llm/openai.py index ddbe5e3d..e3b9000e 100644 --- a/application/llm/openai.py +++ b/application/llm/openai.py @@ -461,15 +461,12 @@ class OpenAILLM(BaseLLM): @staticmethod def _responses_content_parts(role, content): """Translate a cleaned chat ``content`` value into Responses content - parts (``input_text``/``input_image``/``input_file``). - - Assistant text from prior turns is replayed as ``input_text`` too: a - Responses easy-input message only accepts the input_* content types - regardless of role (``output_text`` is reserved for full - output-message items, which carry id/status/annotations). + parts. The Responses API enforces the content-part type by message + role: assistant turns require ``output_text`` (``input_text`` is + rejected with a 400), while user/system turns require ``input_text``. + Images/files use ``input_image``/``input_file``. """ - _ = role - text_type = "input_text" + text_type = "output_text" if role == "assistant" else "input_text" parts = [] if content is None: return parts diff --git a/tests/llm/test_openai_responses.py b/tests/llm/test_openai_responses.py index 67a021d8..127edf49 100644 --- a/tests/llm/test_openai_responses.py +++ b/tests/llm/test_openai_responses.py @@ -102,7 +102,9 @@ def test_to_responses_input_tool_roundtrip(monkeypatch): "arguments": '{"q":"x"}', }, {"type": "function_call_output", "call_id": "call_1", "output": "result"}, - {"role": "assistant", "content": [{"type": "input_text", "text": "final"}]}, + # The Responses API requires output_text (not input_text) for the + # assistant role; input_text 400s. Locked in here so it can't regress. + {"role": "assistant", "content": [{"type": "output_text", "text": "final"}]}, ]