From 4f558e2c3bfc2ec9397fab621e0945e6e4ddffdd Mon Sep 17 00:00:00 2001 From: Bruno Clermont Date: Tue, 14 Jul 2026 21:24:31 -0400 Subject: [PATCH] fix(ollama): resolve context window per model via POST /api/show (#1437) The Ollama integration never applied a correct context window, so agents with real prompts (20k-100k tokens) were rejected with HTTP 400 exceed_context_size_error against a 4096-token default. Three root causes fixed: 1. FetchOllamaModelContext issued a GET to /api/show, which Ollama answers with 405 (the endpoint is POST-only). Now POSTs {"model": ""}. 2. The response parser expected a flat model_info.context_length, but a real Ollama server namespaces the key by architecture (gemma4.context_length, qwen3.context_length, ...). extractContextLength now matches "context_length" or any "*.context_length" key. 3. num_ctx was resolved once at startup for a hardcoded "llama3.3" model and never for the model an agent actually uses. Resolution now happens per request for the real model inside OllamaProvider.resolveNumCtx, cached under an RWMutex, with an explicit settings override winning and the fetched value bounded by OllamaDefaultNumCtx so an enormous advertised window (Qwen3.5 reports 262144) cannot balloon the KV cache beyond VRAM. Also classify Ollama's "exceed_context_size" 400 as a context-overflow error so the pipeline's emergency-compaction+retry path (Issue 958) engages gracefully instead of surfacing a raw error. Co-authored-by: Bruno Clermont Co-authored-by: Claude Opus 4.8 (1M context) --- cmd/gateway_providers.go | 28 ++++----- internal/providers/error_classify.go | 1 + internal/providers/error_classify_test.go | 8 +++ internal/providers/ollama.go | 56 +++++++++++++++--- internal/providers/ollama_context.go | 48 +++++++++++---- internal/providers/ollama_context_test.go | 71 ++++++++++++++++++++++- internal/providers/ollama_test.go | 5 +- 7 files changed, 177 insertions(+), 40 deletions(-) diff --git a/cmd/gateway_providers.go b/cmd/gateway_providers.go index 5a3a8d8b..cc587866 100644 --- a/cmd/gateway_providers.go +++ b/cmd/gateway_providers.go @@ -316,7 +316,7 @@ func registerProvidersFromDB(registry *providers.Registry, provStore store.Provi if host == "" { host = "http://localhost:11434" } - numCtx := resolveOllamaNumCtx(&p, config.DockerLocalhost(host), "") + numCtx := resolveOllamaNumCtx(&p) prov := providers.NewOllamaProvider(p.Name, config.DockerLocalhost(host), "llama3.3", numCtx, nil). WithThinkingEnabled(store.ParseThinkingEnabled(p.Settings)) registry.RegisterForTenant(p.TenantID, prov) @@ -397,7 +397,7 @@ func registerProvidersFromDB(registry *providers.Registry, provStore store.Provi if base == "" { base = "https://ollama.com" } - numCtx := resolveOllamaNumCtx(&p, base, p.APIKey) + numCtx := resolveOllamaNumCtx(&p) prov := providers.NewOllamaProvider(p.Name, base, "llama3.3", numCtx, nil). WithThinkingEnabled(store.ParseThinkingEnabled(p.Settings)) registry.RegisterForTenant(p.TenantID, prov) @@ -466,24 +466,18 @@ func openAIProviderDefaults(providerType, apiBase string) (string, string) { } } -// resolveOllamaNumCtx returns the num_ctx to use for an Ollama provider, or nil -// when the built-in default should be used (provider handles it internally). -// Priority: -// 1. User-configured num_ctx from provider settings JSONB (explicit override wins). -// 2. Value queried from Ollama /api/show for the provider's default model. -// 3. nil when neither is available (OllamaProvider omits options.num_ctx, using Ollama's default). -func resolveOllamaNumCtx(p *store.LLMProviderData, apiBase, apiKey string) *int { +// resolveOllamaNumCtx returns the operator-configured num_ctx for an Ollama +// provider, or nil to let the provider resolve it per model at request time. +// +// Only the explicit settings JSONB override is honoured here. Probing /api/show +// at startup cannot work: the model an agent will use is not known until it +// sends a request, so the probe had to guess a model name, and a wrong guess +// resolved to nothing. OllamaProvider.resolveNumCtx does the lookup against the +// real model instead, and caches it. +func resolveOllamaNumCtx(p *store.LLMProviderData) *int { if s := store.ParseOllamaSettings(p.Settings); s != nil { return s.NumCtx } - // Query the Ollama API for the model's native context length. - // Use a short timeout so startup is not blocked by a slow/absent Ollama server. - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - numCtx := providers.FetchOllamaModelContext(ctx, apiBase, "llama3.3", apiKey) - if numCtx != providers.OllamaDefaultNumCtx { - return &numCtx - } return nil } diff --git a/internal/providers/error_classify.go b/internal/providers/error_classify.go index 44ff5bd3..61df90f3 100644 --- a/internal/providers/error_classify.go +++ b/internal/providers/error_classify.go @@ -169,6 +169,7 @@ func isContextOverflow(lower string) bool { "prompt exceeds max length", // ZAI/GLM-5 "request_too_large", // Generic "input is too long", // DashScope + "exceed_context_size", // Ollama native /api/chat 400 "请求输入过长", // Chinese generic ) } diff --git a/internal/providers/error_classify_test.go b/internal/providers/error_classify_test.go index bb9c3ab7..6a941186 100644 --- a/internal/providers/error_classify_test.go +++ b/internal/providers/error_classify_test.go @@ -111,6 +111,14 @@ func TestClassifyContextWindowExceeded(t *testing.T) { } } +func TestClassifyContextWindowOllama(t *testing.T) { + classifier := NewDefaultClassifier() + result := classifier.Classify(nil, 400, `{"error":"exceed_context_size_error"}`) + if result.Kind != "context_overflow" { + t.Errorf("expected context_overflow kind for Ollama, got %s", result.Kind) + } +} + func TestClassifyContextWindowEnglish(t *testing.T) { classifier := NewDefaultClassifier() result := classifier.Classify(nil, 400, "error: maximum context length reached") diff --git a/internal/providers/ollama.go b/internal/providers/ollama.go index c767d072..b484c494 100644 --- a/internal/providers/ollama.go +++ b/internal/providers/ollama.go @@ -7,6 +7,7 @@ import ( "net/http" "net/url" "strings" + "sync" ollamaapi "github.com/ollama/ollama/api" ) @@ -23,6 +24,12 @@ type OllamaProvider struct { client *ollamaapi.Client retryConfig RetryConfig + // ctxCache memoises each model's context window as reported by /api/show, + // keyed by model name. Resolution happens on the first request for a model + // rather than at startup, because the model an agent uses is only known then. + ctxMu sync.RWMutex + ctxCache map[string]int + // thinkingEnabled is the provider-level override for whether requests // should ask Ollama to emit visible reasoning/thinking tokens. // nil = default off (see buildRequest). @@ -59,9 +66,42 @@ func NewOllamaProvider(name, apiBase, defaultModel string, numCtx *int, httpClie numCtx: numCtx, client: ollamaapi.NewClient(parsedURL, httpClient), retryConfig: DefaultRetryConfig(), + ctxCache: make(map[string]int), } } +// resolveNumCtx returns the context window to request for a given model. +// +// An explicitly configured num_ctx always wins — it is the operator's lever for +// capping the KV cache when a model's full window would not fit in VRAM. +// Otherwise the model's own context length is read from /api/show and cached, +// bounded by OllamaDefaultNumCtx so an enormous advertised window (Qwen3.5 +// reports 262144) cannot balloon the KV cache allocation. +// +// Never returns less than the caller would need: on any lookup failure it falls +// back to OllamaDefaultNumCtx rather than to Ollama's 4096 default, which +// silently rejects any prompt larger than a few thousand tokens. +func (p *OllamaProvider) resolveNumCtx(ctx context.Context, model string) int { + if p.numCtx != nil { + return *p.numCtx + } + + p.ctxMu.RLock() + cached, ok := p.ctxCache[model] + p.ctxMu.RUnlock() + if ok { + return cached + } + + numCtx := min(FetchOllamaModelContext(ctx, p.apiBase, model, p.apiKey), OllamaDefaultNumCtx) + + p.ctxMu.Lock() + p.ctxCache[model] = numCtx + p.ctxMu.Unlock() + + return numCtx +} + // WithThinkingEnabled sets the provider-level override for whether native // Ollama chat requests should ask the model to emit visible reasoning // ("think") tokens. nil (not calling this) preserves the existing default @@ -113,7 +153,7 @@ func (p *OllamaProvider) Chat(ctx context.Context, req ChatRequest) (*ChatRespon } return nil } - if err := p.client.Chat(ctx, p.buildRequest(req, false), streamFn); err != nil { + if err := p.client.Chat(ctx, p.buildRequest(ctx, req, false), streamFn); err != nil { return nil, fmt.Errorf("%s: chat: %w", p.name, err) } if finalResp == nil { @@ -171,7 +211,7 @@ func (p *OllamaProvider) ChatStream(ctx context.Context, req ChatRequest, onChun return nil } - if err := p.client.Chat(ctx, p.buildRequest(req, true), streamFn); err != nil { + if err := p.client.Chat(ctx, p.buildRequest(ctx, req, true), streamFn); err != nil { return nil, fmt.Errorf("%s: chat stream: %w", p.name, err) } return acc, nil @@ -183,7 +223,7 @@ func (p *OllamaProvider) ChatStream(ctx context.Context, req ChatRequest, onChun } // buildRequest converts a generic ChatRequest into an Ollama-native api.ChatRequest. -func (p *OllamaProvider) buildRequest(req ChatRequest, stream bool) *ollamaapi.ChatRequest { +func (p *OllamaProvider) buildRequest(ctx context.Context, req ChatRequest, stream bool) *ollamaapi.ChatRequest { model := req.Model if model == "" { model = p.defaultModel @@ -268,13 +308,11 @@ func (p *OllamaProvider) buildRequest(req ChatRequest, stream bool) *ollamaapi.C // Build options map: num_ctx + caller overrides. // Always set num_ctx so Ollama uses a large context window even when the caller // did not configure a specific value. Without this, Ollama defaults to 4096 and - // rejects conversations that exceed that limit. + // rejects conversations that exceed that limit. Note this only works because the + // request goes to the native /api/chat endpoint — the OpenAI-compat shim at + // /v1/chat/completions drops options.num_ctx on the floor. opts := make(map[string]any) - if p.numCtx != nil { - opts["num_ctx"] = *p.numCtx - } else { - opts["num_ctx"] = OllamaDefaultNumCtx - } + opts["num_ctx"] = p.resolveNumCtx(ctx, model) if temp, ok := req.Options[OptTemperature]; ok { opts["temperature"] = temp } diff --git a/internal/providers/ollama_context.go b/internal/providers/ollama_context.go index c2627421..63a49c83 100644 --- a/internal/providers/ollama_context.go +++ b/internal/providers/ollama_context.go @@ -1,6 +1,7 @@ package providers import ( + "bytes" "context" "encoding/json" "fmt" @@ -28,14 +29,19 @@ func FetchOllamaModelContext(ctx context.Context, apiBase, model, apiKey string) slog.Debug("ollama.context: querying /api/show", "api_base", apiBase, "resolved_base", base, "model", model) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + // /api/show is POST-only: a GET is answered with "405 method not allowed", + // which silently degraded every lookup to the fallback. + payload, err := json.Marshal(map[string]string{"model": model}) + if err != nil { + slog.Warn("ollama.context: encode request failed", "model", model, "error", err, "fallback", OllamaDefaultNumCtx) + return OllamaDefaultNumCtx + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload)) if err != nil { slog.Warn("ollama.context: build request failed", "model", model, "error", err, "fallback", OllamaDefaultNumCtx) return OllamaDefaultNumCtx } - q := req.URL.Query() - q.Set("model", model) - req.URL.RawQuery = q.Encode() + req.Header.Set("Content-Type", "application/json") if apiKey != "" { req.Header.Set("Authorization", "Bearer "+apiKey) } @@ -61,21 +67,41 @@ func FetchOllamaModelContext(ctx context.Context, apiBase, model, apiKey string) slog.Debug("ollama.context: /api/show raw response", "model", model, "response", string(rawBody)) var result struct { - ModelInfo struct { - ContextLength int `json:"context_length"` - } `json:"model_info"` + ModelInfo map[string]json.RawMessage `json:"model_info"` } if err := json.Unmarshal(rawBody, &result); err != nil { slog.Warn("ollama.context: decode failed", "model", model, "error", fmt.Sprintf("%v", err), "fallback", OllamaDefaultNumCtx) return OllamaDefaultNumCtx } - slog.Debug("ollama.context: extracted context_length", "model", model, "context_length", result.ModelInfo.ContextLength) + contextLength := extractContextLength(result.ModelInfo) + slog.Debug("ollama.context: extracted context_length", "model", model, "context_length", contextLength) - if result.ModelInfo.ContextLength <= 0 { + if contextLength <= 0 { slog.Debug("ollama.context: context_length not positive, using default", "model", model, "fallback", OllamaDefaultNumCtx) return OllamaDefaultNumCtx } - slog.Info("ollama.context: resolved context window", "model", model, "num_ctx", result.ModelInfo.ContextLength) - return result.ModelInfo.ContextLength + slog.Info("ollama.context: resolved context window", "model", model, "num_ctx", contextLength) + return contextLength +} + +// extractContextLength pulls the context window out of an /api/show model_info map. +// Ollama namespaces the key by model architecture ("gemma4.context_length", +// "qwen35.context_length", "llama.context_length"), so a fixed "context_length" +// lookup never matches a real server response; the bare key is still accepted +// because it is what hand-written fixtures and older stubs return. +func extractContextLength(modelInfo map[string]json.RawMessage) int { + for key, raw := range modelInfo { + if key != "context_length" && !strings.HasSuffix(key, ".context_length") { + continue + } + var length int + if err := json.Unmarshal(raw, &length); err != nil { + continue + } + if length > 0 { + return length + } + } + return 0 } diff --git a/internal/providers/ollama_context_test.go b/internal/providers/ollama_context_test.go index 9ae2ed21..e4a92cec 100644 --- a/internal/providers/ollama_context_test.go +++ b/internal/providers/ollama_context_test.go @@ -21,7 +21,14 @@ func TestFetchOllamaModelContext_Success(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { require.Equal(t, "/api/show", r.URL.Path) - assert.Equal(t, "llama3", r.URL.Query().Get("model")) + // /api/show is POST-only and takes the model in a JSON body; a GET with a + // query param is answered "405 method not allowed" by a real Ollama server. + assert.Equal(t, http.MethodPost, r.Method) + var body struct { + Model string `json:"model"` + } + require.NoError(t, json.NewDecoder(r.Body).Decode(&body)) + assert.Equal(t, "llama3", body.Model) resp := response{ModelInfo: modelInfo{ContextLength: 8192}} w.Header().Set("Content-Type", "application/json") @@ -33,6 +40,21 @@ func TestFetchOllamaModelContext_Success(t *testing.T) { assert.Equal(t, 8192, got) } +// TestFetchOllamaModelContext_ArchNamespacedKey covers what a real Ollama server +// actually returns: the context length is namespaced by model architecture, never +// exposed under a bare "context_length" key. +func TestFetchOllamaModelContext_ArchNamespacedKey(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, err := w.Write([]byte(`{"model_info":{"general.architecture":"gemma4","gemma4.context_length":131072,"gemma4.block_count":34}}`)) + require.NoError(t, err) + })) + defer srv.Close() + + got := FetchOllamaModelContext(context.Background(), srv.URL, "gemma-4-e4b", "") + assert.Equal(t, 131072, got) +} + func TestFetchOllamaModelContext_SuccessWithV1Suffix(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { assert.Equal(t, "/api/show", r.URL.Path) @@ -111,3 +133,50 @@ func TestFetchOllamaModelContext_ConnectionRefused(t *testing.T) { got := FetchOllamaModelContext(context.Background(), "http://127.0.0.1:19999", "llama3", "") assert.Equal(t, OllamaDefaultNumCtx, got) } + +// TestOllamaProviderResolvesNumCtxPerModel proves the provider asks about the model +// the caller actually requested — not a hardcoded name — and caches the answer. +func TestOllamaProviderResolvesNumCtxPerModel(t *testing.T) { + var calls int + var asked []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + var body struct { + Model string `json:"model"` + } + require.NoError(t, json.NewDecoder(r.Body).Decode(&body)) + asked = append(asked, body.Model) + w.Header().Set("Content-Type", "application/json") + _, err := w.Write([]byte(`{"model_info":{"gemma4.context_length":65536}}`)) + require.NoError(t, err) + })) + defer srv.Close() + + provider := NewOllamaProvider("test", srv.URL, "unused-default", nil, nil) + + req := ChatRequest{Model: "gemma-4-e4b", Messages: []Message{{Role: "user", Content: "hi"}}} + first := provider.buildRequest(context.Background(), req, false) + assert.Equal(t, 65536, first.Options["num_ctx"]) + assert.Equal(t, []string{"gemma-4-e4b"}, asked) + + // Second call for the same model must be served from cache. + second := provider.buildRequest(context.Background(), req, false) + assert.Equal(t, 65536, second.Options["num_ctx"]) + assert.Equal(t, 1, calls, "second request for the same model should hit the cache") +} + +// TestOllamaProviderExplicitNumCtxWins proves the operator's configured value takes +// priority and skips the lookup entirely — the lever for capping KV cache to fit VRAM. +func TestOllamaProviderExplicitNumCtxWins(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { + t.Error("/api/show must not be queried when num_ctx is configured explicitly") + })) + defer srv.Close() + + configured := 16384 + provider := NewOllamaProvider("test", srv.URL, "m", &configured, nil) + + req := ChatRequest{Model: "any-model", Messages: []Message{{Role: "user", Content: "hi"}}} + built := provider.buildRequest(context.Background(), req, false) + assert.Equal(t, 16384, built.Options["num_ctx"]) +} diff --git a/internal/providers/ollama_test.go b/internal/providers/ollama_test.go index d4e6413a..7eb22356 100644 --- a/internal/providers/ollama_test.go +++ b/internal/providers/ollama_test.go @@ -1,6 +1,7 @@ package providers import ( + "context" "testing" "github.com/stretchr/testify/assert" @@ -13,7 +14,7 @@ import ( func TestOllamaBuildRequest_ThinkDefaultsToFalse(t *testing.T) { p := NewOllamaProvider("ollama", "http://localhost:11434", "llama3.3", nil, nil) req := ChatRequest{Messages: []Message{{Role: "user", Content: "hi"}}} - ollamaReq := p.buildRequest(req, false) + ollamaReq := p.buildRequest(context.Background(), req, false) require.NotNil(t, ollamaReq.Think) value, ok := ollamaReq.Think.Value.(bool) @@ -41,7 +42,7 @@ func TestOllamaBuildRequest_ThinkOverride(t *testing.T) { p := NewOllamaProvider("ollama", "http://localhost:11434", "llama3.3", nil, nil). WithThinkingEnabled(tc.override) req := ChatRequest{Messages: []Message{{Role: "user", Content: "hi"}}} - ollamaReq := p.buildRequest(req, false) + ollamaReq := p.buildRequest(context.Background(), req, false) require.NotNil(t, ollamaReq.Think) value, ok := ollamaReq.Think.Value.(bool)