feat: resoning passthrough

This commit is contained in:
Alex committed 2026-06-02 17:27:02 +02:00
1 parent 84630d1314
commit ae6400a709
5 files changed
+198 -1

No files matched your search

+4
View File
@@ -37,6 +37,10 @@ class ModelCapabilities:
context_window: int = 128000
input_cost_per_token: Optional[float] = None
output_cost_per_token: Optional[float] = None
# OpenAI reasoning-model effort hint (none/minimal/low/medium/high/xhigh;
# the accepted subset is model-dependent). Consumed by OpenAILLM; ignored
# by providers that don't accept it.
reasoning_effort: Optional[str] = None
@dataclass
+23
View File
@@ -31,6 +31,15 @@ logger = logging.getLogger(__name__)
BUILTIN_MODELS_DIR = Path(__file__).parent / "models"
DEFAULTS_FILENAME = "_defaults.yaml"
# Accepted reasoning_effort values across the OpenAI reasoning lineup. This
# is the union of all models; the set a given model actually accepts is a
# subset (older o-series take low/medium/high only; GPT-5.5 adds xhigh;
# none/minimal are GPT-5-era additions). Validated at YAML load so a typo
# aborts boot rather than surfacing as a provider 400.
VALID_REASONING_EFFORTS = frozenset(
{"none", "minimal", "low", "medium", "high", "xhigh"}
)
class _DefaultsFile(BaseModel):
"""Schema for ``_defaults.yaml``. Currently just attachment aliases."""
@@ -56,6 +65,17 @@ class _CapabilityFields(BaseModel):
context_window: Optional[int] = None
input_cost_per_token: Optional[float] = None
output_cost_per_token: Optional[float] = None
reasoning_effort: Optional[str] = None
@field_validator("reasoning_effort")
@classmethod
def _valid_reasoning_effort(cls, v: Optional[str]) -> Optional[str]:
if v is not None and v not in VALID_REASONING_EFFORTS:
valid = ", ".join(sorted(VALID_REASONING_EFFORTS))
raise ValueError(
f"reasoning_effort must be one of [{valid}], got {v!r}"
)
return v
class _ModelEntry(_CapabilityFields):
@@ -66,6 +86,7 @@ class _ModelEntry(_CapabilityFields):
description: str = ""
enabled: bool = True
base_url: Optional[str] = None
upstream_model_id: Optional[str] = None
aliases: List[str] = Field(default_factory=list)
@field_validator("id")
@@ -205,6 +226,7 @@ def _build_model(
context_window=pick("context_window", 128000),
input_cost_per_token=pick("input_cost_per_token", None),
output_cost_per_token=pick("output_cost_per_token", None),
reasoning_effort=pick("reasoning_effort", None),
)
return AvailableModel(
@@ -215,6 +237,7 @@ def _build_model(
capabilities=caps,
enabled=entry.enabled,
base_url=entry.base_url,
upstream_model_id=entry.upstream_model_id,
display_provider=display_provider,
)
+38 -1
View File
@@ -110,16 +110,53 @@ defaults: # optional, applied to every model below
context_window: int # default 128000
input_cost_per_token: float # default null
output_cost_per_token: float # default null
reasoning_effort: <string> # default null; none|minimal|low|medium|high|xhigh (subset is model-dependent)
models: # required
- id: <string, required> # the value persisted in agent records
- id: <string, required> # unique registry key; persisted in agent records
display_name: <string> # default: id
description: <string> # default: ""
enabled: bool # default true; false hides from /api/models
base_url: <string> # optional custom endpoint for this model
upstream_model_id: <string> # default: id; the name actually sent to the provider
# All `defaults:` fields above can be overridden here per-model.
```
### Reasoning effort, and one model at multiple efforts
`reasoning_effort` is forwarded to the provider for OpenAI reasoning
models. Accepted values are `none`, `minimal`, `low`, `medium`, `high`,
and `xhigh`, but the subset each model accepts varies (older o-series
take only `low`/`medium`/`high`; GPT-5.5 adds `xhigh`) — check the model
page. Set it per-model; sending it to a non-reasoning model is rejected
by the API:
```yaml
- id: gpt-5.4-mini
display_name: GPT-5.4 Mini
reasoning_effort: medium
```
To expose the *same* upstream model at two efforts, give each entry a
distinct `id` and point both at one `upstream_model_id`. The `id` is the
unique registry key (and what's stored in agent records); the
`upstream_model_id` is the name actually sent to the provider, defaulting
to `id` when omitted:
```yaml
- id: gpt-5.4-mini-low
display_name: GPT-5.4 Mini (Low Reasoning)
upstream_model_id: gpt-5.4-mini
reasoning_effort: low
- id: gpt-5.4-mini-high
display_name: GPT-5.4 Mini (High Reasoning)
upstream_model_id: gpt-5.4-mini
reasoning_effort: high
```
Both call `gpt-5.4-mini` on the wire; token usage is attributed to the
distinct `id`s, so cost dashboards split by reasoning level.
### Attachment aliases
The `attachments:` list can mix human-readable aliases with raw MIME
+20
View File
@@ -283,6 +283,8 @@ class OpenAILLM(BaseLLM):
if response_format and not self._supports_structured_output():
response_format = None
self._apply_reasoning_effort(kwargs)
request_params = {
"model": model,
"messages": messages,
@@ -326,6 +328,8 @@ class OpenAILLM(BaseLLM):
if response_format and not self._supports_structured_output():
response_format = None
self._apply_reasoning_effort(kwargs)
request_params = {
"model": model,
"messages": messages,
@@ -380,6 +384,22 @@ class OpenAILLM(BaseLLM):
return bool(self.capabilities.supports_structured_output)
return True
def _apply_reasoning_effort(self, kwargs):
"""Inject the model's configured reasoning_effort into ``kwargs``.
No-op when the caller already set one, when no registry capabilities
are attached, or when the model has no configured effort. Read from
per-model capabilities (not the caller) so a cross-provider fallback
applies its own model's effort rather than inheriting the primary's.
"""
if "reasoning_effort" in kwargs:
return
if self.capabilities is None:
return
effort = getattr(self.capabilities, "reasoning_effort", None)
if effort:
kwargs["reasoning_effort"] = effort
def prepare_structured_output_format(self, json_schema):
if not json_schema:
return None
+113
View File
@@ -111,3 +111,116 @@ def test_model_validation_error_message():
error_msg = str(exc_info.value)
assert "Invalid model_id 'invalid-model-xyz'" in error_msg
assert "Available models:" in error_msg
@pytest.mark.unit
def test_capabilities_reasoning_effort_defaults_none():
"""reasoning_effort is an optional capability, None by default."""
assert ModelCapabilities().reasoning_effort is None
@pytest.mark.unit
def test_yaml_reasoning_effort_and_upstream_model_id(tmp_path):
"""Two distinct ids can share one upstream model, each with its own effort."""
from application.core.model_yaml import load_model_yamls
(tmp_path / "openai.yaml").write_text(
"provider: openai\n"
"models:\n"
" - id: mini-low\n"
" upstream_model_id: mini\n"
" reasoning_effort: low\n"
" - id: mini-high\n"
" upstream_model_id: mini\n"
" reasoning_effort: high\n"
" - id: plain\n",
encoding="utf-8",
)
catalogs = load_model_yamls([tmp_path])
models = {m.id: m for c in catalogs for m in c.models}
assert models["mini-low"].upstream_model_id == "mini"
assert models["mini-low"].capabilities.reasoning_effort == "low"
assert models["mini-high"].upstream_model_id == "mini"
assert models["mini-high"].capabilities.reasoning_effort == "high"
# No upstream_model_id / reasoning_effort given → fall back to id / None.
assert models["plain"].upstream_model_id is None
assert models["plain"].capabilities.reasoning_effort is None
@pytest.mark.unit
def test_yaml_invalid_reasoning_effort_rejected(tmp_path):
"""A bad reasoning_effort value aborts the YAML load."""
from application.core.model_yaml import ModelYAMLError, load_model_yamls
(tmp_path / "openai.yaml").write_text(
"provider: openai\n"
"models:\n"
" - id: bad\n"
" reasoning_effort: turbo\n",
encoding="utf-8",
)
with pytest.raises(ModelYAMLError):
load_model_yamls([tmp_path])
@pytest.mark.unit
def test_yaml_reasoning_effort_accepts_full_enum(tmp_path):
"""Every value OpenAI documents across the GPT-5 series must parse."""
from application.core.model_yaml import (
VALID_REASONING_EFFORTS,
load_model_yamls,
)
assert VALID_REASONING_EFFORTS == {
"none",
"minimal",
"low",
"medium",
"high",
"xhigh",
}
lines = ["provider: openai", "models:"]
for effort in sorted(VALID_REASONING_EFFORTS):
lines.append(f" - id: m-{effort}")
lines.append(f" reasoning_effort: {effort}")
(tmp_path / "openai.yaml").write_text("\n".join(lines) + "\n", encoding="utf-8")
catalogs = load_model_yamls([tmp_path])
parsed = {m.id: m.capabilities.reasoning_effort for c in catalogs for m in c.models}
for effort in VALID_REASONING_EFFORTS:
assert parsed[f"m-{effort}"] == effort
@pytest.mark.unit
def test_openai_apply_reasoning_effort():
"""OpenAILLM injects reasoning_effort from capabilities; caller wins."""
from application.llm.openai import OpenAILLM
llm = OpenAILLM.__new__(OpenAILLM)
# Pulled from capabilities when the caller didn't set one.
llm.capabilities = ModelCapabilities(reasoning_effort="high")
kwargs: dict = {}
llm._apply_reasoning_effort(kwargs)
assert kwargs["reasoning_effort"] == "high"
# A caller-supplied value is never overridden.
kwargs = {"reasoning_effort": "low"}
llm._apply_reasoning_effort(kwargs)
assert kwargs["reasoning_effort"] == "low"
# No capabilities / no configured effort → key is not added.
llm.capabilities = None
kwargs = {}
llm._apply_reasoning_effort(kwargs)
assert "reasoning_effort" not in kwargs
llm.capabilities = ModelCapabilities()
kwargs = {}
llm._apply_reasoning_effort(kwargs)
assert "reasoning_effort" not in kwargs