From 23d0b5eb0b02417cabdd1e602cf6e6047e6f9eb0 Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Thu, 19 Mar 2026 08:41:20 +0700 Subject: [PATCH] fix(providers): auto-clamp max_tokens on model rejection (#267) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(providers): auto-clamp max_tokens on model rejection + fix verify for reasoning models When OpenAI-compat models reject max_tokens as too large (e.g. gpt-3.5-turbo supports 4096 but we send 8192), parse the model's stated limit from the 400 error, clamp the value, and retry once. This fixes agent creation for models with lower output token limits without hardcoding model names. Also increase the provider verify endpoint's max_tokens from 1 to 50 so reasoning models (gpt-5, o-series) have enough headroom for internal reasoning during the check call. Closes #248, closes #245 * refactor(providers): extract chat retry closure + fix clamp log key - Extract duplicate retry closure into chatRequestFn() to follow DRY - Fix slog logging wrong key: body["max_tokens"] was nil for reasoning models that use max_completion_tokens — now uses clampedLimit() helper - Remove unnecessary _ = resp in provider verify endpoint --------- Co-authored-by: viettranx --- internal/http/provider_verify.go | 4 +- internal/providers/openai.go | 76 +++++++++++++++++++++++++++++++- 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/internal/http/provider_verify.go b/internal/http/provider_verify.go index 7aecfa1e..161dea9a 100644 --- a/internal/http/provider_verify.go +++ b/internal/http/provider_verify.go @@ -104,14 +104,14 @@ func (h *ProvidersHandler) handleVerifyProvider(w http.ResponseWriter, r *http.R }, Model: req.Model, Options: map[string]any{ - "max_tokens": 1, + // Use a small but safe value — reasoning models need headroom beyond 1 token. + "max_tokens": 50, }, }) if err != nil { writeJSON(w, http.StatusOK, map[string]any{"valid": false, "error": friendlyVerifyError(err)}) return } - writeJSON(w, http.StatusOK, map[string]any{"valid": true}) } diff --git a/internal/providers/openai.go b/internal/providers/openai.go index fdb182b2..42ebc1bd 100644 --- a/internal/providers/openai.go +++ b/internal/providers/openai.go @@ -5,10 +5,13 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "log/slog" "net/http" + "regexp" + "strconv" "strings" "time" ) @@ -79,7 +82,25 @@ func (p *OpenAIProvider) Chat(ctx context.Context, req ChatRequest) (*ChatRespon model := p.resolveModel(req.Model) body := p.buildRequestBody(model, req, false) - return RetryDo(ctx, p.retryConfig, func() (*ChatResponse, error) { + chatFn := p.chatRequestFn(ctx, body) + + resp, err := RetryDo(ctx, p.retryConfig, chatFn) + + // Auto-clamp max_tokens and retry once if the model rejects the value + if err != nil { + if clamped := clampMaxTokensFromError(err, body); clamped { + slog.Info("max_tokens clamped, retrying", "model", model, "limit", clampedLimit(body)) + return RetryDo(ctx, p.retryConfig, chatFn) + } + } + + return resp, err +} + +// chatRequestFn returns a closure that performs a single non-streaming chat request. +// Shared between initial attempt and post-clamp retry to avoid duplication. +func (p *OpenAIProvider) chatRequestFn(ctx context.Context, body map[string]any) func() (*ChatResponse, error) { + return func() (*ChatResponse, error) { respBody, err := p.doRequest(ctx, body) if err != nil { return nil, err @@ -92,7 +113,7 @@ func (p *OpenAIProvider) Chat(ctx context.Context, req ChatRequest) (*ChatRespon } return p.parseResponse(&oaiResp), nil - }) + } } func (p *OpenAIProvider) ChatStream(ctx context.Context, req ChatRequest, onChunk func(StreamChunk)) (*ChatResponse, error) { @@ -103,6 +124,16 @@ func (p *OpenAIProvider) ChatStream(ctx context.Context, req ChatRequest, onChun respBody, err := RetryDo(ctx, p.retryConfig, func() (io.ReadCloser, error) { return p.doRequest(ctx, body) }) + + // Auto-clamp max_tokens and retry once if the model rejects the value + if err != nil { + if clamped := clampMaxTokensFromError(err, body); clamped { + slog.Info("max_tokens clamped, retrying stream", "model", model, "limit", clampedLimit(body)) + respBody, err = RetryDo(ctx, p.retryConfig, func() (io.ReadCloser, error) { + return p.doRequest(ctx, body) + }) + } + } if err != nil { return nil, err } @@ -438,3 +469,44 @@ func (p *OpenAIProvider) parseResponse(resp *openAIResponse) *ChatResponse { return result } + +// maxTokensLimitRe matches "supports at most N completion tokens" from OpenAI 400 errors. +var maxTokensLimitRe = regexp.MustCompile(`supports at most (\d+) completion tokens`) + +// clampMaxTokensFromError checks if an error is a 400 "max_tokens is too large" rejection. +// If so, it parses the model's stated limit, clamps the body's max_tokens/max_completion_tokens, +// and returns true so the caller can retry. +func clampMaxTokensFromError(err error, body map[string]any) bool { + var httpErr *HTTPError + if !errors.As(err, &httpErr) || httpErr.Status != http.StatusBadRequest { + return false + } + if !strings.Contains(httpErr.Body, "max_tokens") || !strings.Contains(httpErr.Body, "too large") { + return false + } + + matches := maxTokensLimitRe.FindStringSubmatch(httpErr.Body) + if len(matches) < 2 { + return false + } + limit, parseErr := strconv.Atoi(matches[1]) + if parseErr != nil || limit <= 0 { + return false + } + + // Clamp whichever key is present + if _, ok := body["max_completion_tokens"]; ok { + body["max_completion_tokens"] = limit + } else { + body["max_tokens"] = limit + } + return true +} + +// clampedLimit returns the clamped max_tokens or max_completion_tokens value for logging. +func clampedLimit(body map[string]any) any { + if v, ok := body["max_completion_tokens"]; ok { + return v + } + return body["max_tokens"] +}