diff --git a/application/core/model_settings.py b/application/core/model_settings.py index 49843e57..17664f04 100644 --- a/application/core/model_settings.py +++ b/application/core/model_settings.py @@ -37,6 +37,10 @@ class ModelCapabilities: context_window: int = 128000 input_cost_per_token: Optional[float] = None output_cost_per_token: Optional[float] = None + # OpenAI reasoning-model effort hint (none/minimal/low/medium/high/xhigh; + # the accepted subset is model-dependent). Consumed by OpenAILLM; ignored + # by providers that don't accept it. + reasoning_effort: Optional[str] = None @dataclass diff --git a/application/core/model_yaml.py b/application/core/model_yaml.py index eb666c68..3b5794ef 100644 --- a/application/core/model_yaml.py +++ b/application/core/model_yaml.py @@ -31,6 +31,15 @@ logger = logging.getLogger(__name__) BUILTIN_MODELS_DIR = Path(__file__).parent / "models" DEFAULTS_FILENAME = "_defaults.yaml" +# Accepted reasoning_effort values across the OpenAI reasoning lineup. This +# is the union of all models; the set a given model actually accepts is a +# subset (older o-series take low/medium/high only; GPT-5.5 adds xhigh; +# none/minimal are GPT-5-era additions). Validated at YAML load so a typo +# aborts boot rather than surfacing as a provider 400. +VALID_REASONING_EFFORTS = frozenset( + {"none", "minimal", "low", "medium", "high", "xhigh"} +) + class _DefaultsFile(BaseModel): """Schema for ``_defaults.yaml``. Currently just attachment aliases.""" @@ -56,6 +65,17 @@ class _CapabilityFields(BaseModel): context_window: Optional[int] = None input_cost_per_token: Optional[float] = None output_cost_per_token: Optional[float] = None + reasoning_effort: Optional[str] = None + + @field_validator("reasoning_effort") + @classmethod + def _valid_reasoning_effort(cls, v: Optional[str]) -> Optional[str]: + if v is not None and v not in VALID_REASONING_EFFORTS: + valid = ", ".join(sorted(VALID_REASONING_EFFORTS)) + raise ValueError( + f"reasoning_effort must be one of [{valid}], got {v!r}" + ) + return v class _ModelEntry(_CapabilityFields): @@ -66,6 +86,7 @@ class _ModelEntry(_CapabilityFields): description: str = "" enabled: bool = True base_url: Optional[str] = None + upstream_model_id: Optional[str] = None aliases: List[str] = Field(default_factory=list) @field_validator("id") @@ -205,6 +226,7 @@ def _build_model( context_window=pick("context_window", 128000), input_cost_per_token=pick("input_cost_per_token", None), output_cost_per_token=pick("output_cost_per_token", None), + reasoning_effort=pick("reasoning_effort", None), ) return AvailableModel( @@ -215,6 +237,7 @@ def _build_model( capabilities=caps, enabled=entry.enabled, base_url=entry.base_url, + upstream_model_id=entry.upstream_model_id, display_provider=display_provider, ) diff --git a/application/core/models/README.md b/application/core/models/README.md index f504591f..6d45323f 100644 --- a/application/core/models/README.md +++ b/application/core/models/README.md @@ -110,16 +110,53 @@ defaults: # optional, applied to every model below context_window: int # default 128000 input_cost_per_token: float # default null output_cost_per_token: float # default null + reasoning_effort: # default null; none|minimal|low|medium|high|xhigh (subset is model-dependent) models: # required - - id: # the value persisted in agent records + - id: # unique registry key; persisted in agent records display_name: # default: id description: # default: "" enabled: bool # default true; false hides from /api/models base_url: # optional custom endpoint for this model + upstream_model_id: # default: id; the name actually sent to the provider # All `defaults:` fields above can be overridden here per-model. ``` +### Reasoning effort, and one model at multiple efforts + +`reasoning_effort` is forwarded to the provider for OpenAI reasoning +models. Accepted values are `none`, `minimal`, `low`, `medium`, `high`, +and `xhigh`, but the subset each model accepts varies (older o-series +take only `low`/`medium`/`high`; GPT-5.5 adds `xhigh`) — check the model +page. Set it per-model; sending it to a non-reasoning model is rejected +by the API: + +```yaml + - id: gpt-5.4-mini + display_name: GPT-5.4 Mini + reasoning_effort: medium +``` + +To expose the *same* upstream model at two efforts, give each entry a +distinct `id` and point both at one `upstream_model_id`. The `id` is the +unique registry key (and what's stored in agent records); the +`upstream_model_id` is the name actually sent to the provider, defaulting +to `id` when omitted: + +```yaml + - id: gpt-5.4-mini-low + display_name: GPT-5.4 Mini (Low Reasoning) + upstream_model_id: gpt-5.4-mini + reasoning_effort: low + - id: gpt-5.4-mini-high + display_name: GPT-5.4 Mini (High Reasoning) + upstream_model_id: gpt-5.4-mini + reasoning_effort: high +``` + +Both call `gpt-5.4-mini` on the wire; token usage is attributed to the +distinct `id`s, so cost dashboards split by reasoning level. + ### Attachment aliases The `attachments:` list can mix human-readable aliases with raw MIME diff --git a/application/llm/openai.py b/application/llm/openai.py index 69228d01..4b316ffb 100644 --- a/application/llm/openai.py +++ b/application/llm/openai.py @@ -283,6 +283,8 @@ class OpenAILLM(BaseLLM): if response_format and not self._supports_structured_output(): response_format = None + self._apply_reasoning_effort(kwargs) + request_params = { "model": model, "messages": messages, @@ -326,6 +328,8 @@ class OpenAILLM(BaseLLM): if response_format and not self._supports_structured_output(): response_format = None + self._apply_reasoning_effort(kwargs) + request_params = { "model": model, "messages": messages, @@ -380,6 +384,22 @@ class OpenAILLM(BaseLLM): return bool(self.capabilities.supports_structured_output) return True + def _apply_reasoning_effort(self, kwargs): + """Inject the model's configured reasoning_effort into ``kwargs``. + + No-op when the caller already set one, when no registry capabilities + are attached, or when the model has no configured effort. Read from + per-model capabilities (not the caller) so a cross-provider fallback + applies its own model's effort rather than inheriting the primary's. + """ + if "reasoning_effort" in kwargs: + return + if self.capabilities is None: + return + effort = getattr(self.capabilities, "reasoning_effort", None) + if effort: + kwargs["reasoning_effort"] = effort + def prepare_structured_output_format(self, json_schema): if not json_schema: return None diff --git a/tests/test_model_validation.py b/tests/test_model_validation.py index 3a5eeb3c..a7668242 100644 --- a/tests/test_model_validation.py +++ b/tests/test_model_validation.py @@ -111,3 +111,116 @@ def test_model_validation_error_message(): error_msg = str(exc_info.value) assert "Invalid model_id 'invalid-model-xyz'" in error_msg assert "Available models:" in error_msg + + +@pytest.mark.unit +def test_capabilities_reasoning_effort_defaults_none(): + """reasoning_effort is an optional capability, None by default.""" + assert ModelCapabilities().reasoning_effort is None + + +@pytest.mark.unit +def test_yaml_reasoning_effort_and_upstream_model_id(tmp_path): + """Two distinct ids can share one upstream model, each with its own effort.""" + from application.core.model_yaml import load_model_yamls + + (tmp_path / "openai.yaml").write_text( + "provider: openai\n" + "models:\n" + " - id: mini-low\n" + " upstream_model_id: mini\n" + " reasoning_effort: low\n" + " - id: mini-high\n" + " upstream_model_id: mini\n" + " reasoning_effort: high\n" + " - id: plain\n", + encoding="utf-8", + ) + + catalogs = load_model_yamls([tmp_path]) + models = {m.id: m for c in catalogs for m in c.models} + + assert models["mini-low"].upstream_model_id == "mini" + assert models["mini-low"].capabilities.reasoning_effort == "low" + assert models["mini-high"].upstream_model_id == "mini" + assert models["mini-high"].capabilities.reasoning_effort == "high" + + # No upstream_model_id / reasoning_effort given → fall back to id / None. + assert models["plain"].upstream_model_id is None + assert models["plain"].capabilities.reasoning_effort is None + + +@pytest.mark.unit +def test_yaml_invalid_reasoning_effort_rejected(tmp_path): + """A bad reasoning_effort value aborts the YAML load.""" + from application.core.model_yaml import ModelYAMLError, load_model_yamls + + (tmp_path / "openai.yaml").write_text( + "provider: openai\n" + "models:\n" + " - id: bad\n" + " reasoning_effort: turbo\n", + encoding="utf-8", + ) + + with pytest.raises(ModelYAMLError): + load_model_yamls([tmp_path]) + + +@pytest.mark.unit +def test_yaml_reasoning_effort_accepts_full_enum(tmp_path): + """Every value OpenAI documents across the GPT-5 series must parse.""" + from application.core.model_yaml import ( + VALID_REASONING_EFFORTS, + load_model_yamls, + ) + + assert VALID_REASONING_EFFORTS == { + "none", + "minimal", + "low", + "medium", + "high", + "xhigh", + } + + lines = ["provider: openai", "models:"] + for effort in sorted(VALID_REASONING_EFFORTS): + lines.append(f" - id: m-{effort}") + lines.append(f" reasoning_effort: {effort}") + (tmp_path / "openai.yaml").write_text("\n".join(lines) + "\n", encoding="utf-8") + + catalogs = load_model_yamls([tmp_path]) + parsed = {m.id: m.capabilities.reasoning_effort for c in catalogs for m in c.models} + for effort in VALID_REASONING_EFFORTS: + assert parsed[f"m-{effort}"] == effort + + +@pytest.mark.unit +def test_openai_apply_reasoning_effort(): + """OpenAILLM injects reasoning_effort from capabilities; caller wins.""" + from application.llm.openai import OpenAILLM + + llm = OpenAILLM.__new__(OpenAILLM) + + # Pulled from capabilities when the caller didn't set one. + llm.capabilities = ModelCapabilities(reasoning_effort="high") + kwargs: dict = {} + llm._apply_reasoning_effort(kwargs) + assert kwargs["reasoning_effort"] == "high" + + # A caller-supplied value is never overridden. + kwargs = {"reasoning_effort": "low"} + llm._apply_reasoning_effort(kwargs) + assert kwargs["reasoning_effort"] == "low" + + # No capabilities / no configured effort → key is not added. + llm.capabilities = None + kwargs = {} + llm._apply_reasoning_effort(kwargs) + assert "reasoning_effort" not in kwargs + + llm.capabilities = ModelCapabilities() + kwargs = {} + llm._apply_reasoning_effort(kwargs) + assert "reasoning_effort" not in kwargs