fix(ollama): resolve context window per model via POST /api/show (#1437)

The Ollama integration never applied a correct context window, so agents
with real prompts (20k-100k tokens) were rejected with HTTP 400
exceed_context_size_error against a 4096-token default.

Three root causes fixed:

1. FetchOllamaModelContext issued a GET to /api/show, which Ollama answers
   with 405 (the endpoint is POST-only). Now POSTs {"model": "<name>"}.

2. The response parser expected a flat model_info.context_length, but a real
   Ollama server namespaces the key by architecture (gemma4.context_length,
   qwen3.context_length, ...). extractContextLength now matches "context_length"
   or any "*.context_length" key.

3. num_ctx was resolved once at startup for a hardcoded "llama3.3" model and
   never for the model an agent actually uses. Resolution now happens per
   request for the real model inside OllamaProvider.resolveNumCtx, cached under
   an RWMutex, with an explicit settings override winning and the fetched value
   bounded by OllamaDefaultNumCtx so an enormous advertised window (Qwen3.5
   reports 262144) cannot balloon the KV cache beyond VRAM.

Also classify Ollama's "exceed_context_size" 400 as a context-overflow error so
the pipeline's emergency-compaction+retry path (Issue 958) engages gracefully
instead of surfacing a raw error.

Co-authored-by: Bruno Clermont <bruno.clermont@gmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

cmd/gateway_providers.go

@@ -316,7 +316,7 @@ func registerProvidersFromDB(registry *providers.Registry, provStore store.Provi
 			if host == "" {
 				host = "http://localhost:11434"
 			}
-			numCtx := resolveOllamaNumCtx(&p, config.DockerLocalhost(host), "")
+			numCtx := resolveOllamaNumCtx(&p)
 			prov := providers.NewOllamaProvider(p.Name, config.DockerLocalhost(host), "llama3.3", numCtx, nil).
 				WithThinkingEnabled(store.ParseThinkingEnabled(p.Settings))
 			registry.RegisterForTenant(p.TenantID, prov)
@@ -397,7 +397,7 @@ func registerProvidersFromDB(registry *providers.Registry, provStore store.Provi
 			if base == "" {
 				base = "https://ollama.com"
 			}
-			numCtx := resolveOllamaNumCtx(&p, base, p.APIKey)
+			numCtx := resolveOllamaNumCtx(&p)
 			prov := providers.NewOllamaProvider(p.Name, base, "llama3.3", numCtx, nil).
 				WithThinkingEnabled(store.ParseThinkingEnabled(p.Settings))
 			registry.RegisterForTenant(p.TenantID, prov)
@@ -466,24 +466,18 @@ func openAIProviderDefaults(providerType, apiBase string) (string, string) {
 	}
 }
 
-// resolveOllamaNumCtx returns the num_ctx to use for an Ollama provider, or nil
-// when the built-in default should be used (provider handles it internally).
-// Priority:
-//  1. User-configured num_ctx from provider settings JSONB (explicit override wins).
-//  2. Value queried from Ollama /api/show for the provider's default model.
-//  3. nil when neither is available (OllamaProvider omits options.num_ctx, using Ollama's default).
-func resolveOllamaNumCtx(p *store.LLMProviderData, apiBase, apiKey string) *int {
+// resolveOllamaNumCtx returns the operator-configured num_ctx for an Ollama
+// provider, or nil to let the provider resolve it per model at request time.
+//
+// Only the explicit settings JSONB override is honoured here. Probing /api/show
+// at startup cannot work: the model an agent will use is not known until it
+// sends a request, so the probe had to guess a model name, and a wrong guess
+// resolved to nothing. OllamaProvider.resolveNumCtx does the lookup against the
+// real model instead, and caches it.
+func resolveOllamaNumCtx(p *store.LLMProviderData) *int {
 	if s := store.ParseOllamaSettings(p.Settings); s != nil {
 		return s.NumCtx
 	}
-	// Query the Ollama API for the model's native context length.
-	// Use a short timeout so startup is not blocked by a slow/absent Ollama server.
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-	numCtx := providers.FetchOllamaModelContext(ctx, apiBase, "llama3.3", apiKey)
-	if numCtx != providers.OllamaDefaultNumCtx {
-		return &numCtx
-	}
 	return nil
 }
 

internal/providers/error_classify.go

@@ -169,6 +169,7 @@ func isContextOverflow(lower string) bool {
 		"prompt exceeds max length", // ZAI/GLM-5
 		"request_too_large",         // Generic
 		"input is too long",         // DashScope
+		"exceed_context_size",       // Ollama native /api/chat 400
 		"请求输入过长",                    // Chinese generic
 	)
 }

internal/providers/error_classify_test.go

@@ -111,6 +111,14 @@ func TestClassifyContextWindowExceeded(t *testing.T) {
 	}
 }
 
+func TestClassifyContextWindowOllama(t *testing.T) {
+	classifier := NewDefaultClassifier()
+	result := classifier.Classify(nil, 400, `{"error":"exceed_context_size_error"}`)
+	if result.Kind != "context_overflow" {
+		t.Errorf("expected context_overflow kind for Ollama, got %s", result.Kind)
+	}
+}
+
 func TestClassifyContextWindowEnglish(t *testing.T) {
 	classifier := NewDefaultClassifier()
 	result := classifier.Classify(nil, 400, "error: maximum context length reached")

internal/providers/ollama.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"net/url"
 	"strings"
+	"sync"
 
 	ollamaapi "github.com/ollama/ollama/api"
 )
@@ -23,6 +24,12 @@ type OllamaProvider struct {
 	client       *ollamaapi.Client
 	retryConfig  RetryConfig
 
+	// ctxCache memoises each model's context window as reported by /api/show,
+	// keyed by model name. Resolution happens on the first request for a model
+	// rather than at startup, because the model an agent uses is only known then.
+	ctxMu    sync.RWMutex
+	ctxCache map[string]int
+
 	// thinkingEnabled is the provider-level override for whether requests
 	// should ask Ollama to emit visible reasoning/thinking tokens.
 	// nil = default off (see buildRequest).
@@ -59,9 +66,42 @@ func NewOllamaProvider(name, apiBase, defaultModel string, numCtx *int, httpClie
 		numCtx:       numCtx,
 		client:       ollamaapi.NewClient(parsedURL, httpClient),
 		retryConfig:  DefaultRetryConfig(),
+		ctxCache:     make(map[string]int),
 	}
 }
 
+// resolveNumCtx returns the context window to request for a given model.
+//
+// An explicitly configured num_ctx always wins — it is the operator's lever for
+// capping the KV cache when a model's full window would not fit in VRAM.
+// Otherwise the model's own context length is read from /api/show and cached,
+// bounded by OllamaDefaultNumCtx so an enormous advertised window (Qwen3.5
+// reports 262144) cannot balloon the KV cache allocation.
+//
+// Never returns less than the caller would need: on any lookup failure it falls
+// back to OllamaDefaultNumCtx rather than to Ollama's 4096 default, which
+// silently rejects any prompt larger than a few thousand tokens.
+func (p *OllamaProvider) resolveNumCtx(ctx context.Context, model string) int {
+	if p.numCtx != nil {
+		return *p.numCtx
+	}
+
+	p.ctxMu.RLock()
+	cached, ok := p.ctxCache[model]
+	p.ctxMu.RUnlock()
+	if ok {
+		return cached
+	}
+
+	numCtx := min(FetchOllamaModelContext(ctx, p.apiBase, model, p.apiKey), OllamaDefaultNumCtx)
+
+	p.ctxMu.Lock()
+	p.ctxCache[model] = numCtx
+	p.ctxMu.Unlock()
+
+	return numCtx
+}
+
 // WithThinkingEnabled sets the provider-level override for whether native
 // Ollama chat requests should ask the model to emit visible reasoning
 // ("think") tokens. nil (not calling this) preserves the existing default
@@ -113,7 +153,7 @@ func (p *OllamaProvider) Chat(ctx context.Context, req ChatRequest) (*ChatRespon
 			}
 			return nil
 		}
-		if err := p.client.Chat(ctx, p.buildRequest(req, false), streamFn); err != nil {
+		if err := p.client.Chat(ctx, p.buildRequest(ctx, req, false), streamFn); err != nil {
 			return nil, fmt.Errorf("%s: chat: %w", p.name, err)
 		}
 		if finalResp == nil {
@@ -171,7 +211,7 @@ func (p *OllamaProvider) ChatStream(ctx context.Context, req ChatRequest, onChun
 			return nil
 		}
 
-		if err := p.client.Chat(ctx, p.buildRequest(req, true), streamFn); err != nil {
+		if err := p.client.Chat(ctx, p.buildRequest(ctx, req, true), streamFn); err != nil {
 			return nil, fmt.Errorf("%s: chat stream: %w", p.name, err)
 		}
 		return acc, nil
@@ -183,7 +223,7 @@ func (p *OllamaProvider) ChatStream(ctx context.Context, req ChatRequest, onChun
 }
 
 // buildRequest converts a generic ChatRequest into an Ollama-native api.ChatRequest.
-func (p *OllamaProvider) buildRequest(req ChatRequest, stream bool) *ollamaapi.ChatRequest {
+func (p *OllamaProvider) buildRequest(ctx context.Context, req ChatRequest, stream bool) *ollamaapi.ChatRequest {
 	model := req.Model
 	if model == "" {
 		model = p.defaultModel
@@ -268,13 +308,11 @@ func (p *OllamaProvider) buildRequest(req ChatRequest, stream bool) *ollamaapi.C
 	// Build options map: num_ctx + caller overrides.
 	// Always set num_ctx so Ollama uses a large context window even when the caller
 	// did not configure a specific value. Without this, Ollama defaults to 4096 and
-	// rejects conversations that exceed that limit.
+	// rejects conversations that exceed that limit. Note this only works because the
+	// request goes to the native /api/chat endpoint — the OpenAI-compat shim at
+	// /v1/chat/completions drops options.num_ctx on the floor.
 	opts := make(map[string]any)
-	if p.numCtx != nil {
-		opts["num_ctx"] = *p.numCtx
-	} else {
-		opts["num_ctx"] = OllamaDefaultNumCtx
-	}
+	opts["num_ctx"] = p.resolveNumCtx(ctx, model)
 	if temp, ok := req.Options[OptTemperature]; ok {
 		opts["temperature"] = temp
 	}

internal/providers/ollama_context.go

@@ -1,6 +1,7 @@
 package providers
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -28,14 +29,19 @@ func FetchOllamaModelContext(ctx context.Context, apiBase, model, apiKey string)
 
 	slog.Debug("ollama.context: querying /api/show", "api_base", apiBase, "resolved_base", base, "model", model)
 
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	// /api/show is POST-only: a GET is answered with "405 method not allowed",
+	// which silently degraded every lookup to the fallback.
+	payload, err := json.Marshal(map[string]string{"model": model})
+	if err != nil {
+		slog.Warn("ollama.context: encode request failed", "model", model, "error", err, "fallback", OllamaDefaultNumCtx)
+		return OllamaDefaultNumCtx
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload))
 	if err != nil {
 		slog.Warn("ollama.context: build request failed", "model", model, "error", err, "fallback", OllamaDefaultNumCtx)
 		return OllamaDefaultNumCtx
 	}
-	q := req.URL.Query()
-	q.Set("model", model)
-	req.URL.RawQuery = q.Encode()
+	req.Header.Set("Content-Type", "application/json")
 	if apiKey != "" {
 		req.Header.Set("Authorization", "Bearer "+apiKey)
 	}
@@ -61,21 +67,41 @@ func FetchOllamaModelContext(ctx context.Context, apiBase, model, apiKey string)
 	slog.Debug("ollama.context: /api/show raw response", "model", model, "response", string(rawBody))
 
 	var result struct {
-		ModelInfo struct {
-			ContextLength int `json:"context_length"`
-		} `json:"model_info"`
+		ModelInfo map[string]json.RawMessage `json:"model_info"`
 	}
 	if err := json.Unmarshal(rawBody, &result); err != nil {
 		slog.Warn("ollama.context: decode failed", "model", model, "error", fmt.Sprintf("%v", err), "fallback", OllamaDefaultNumCtx)
 		return OllamaDefaultNumCtx
 	}
 
-	slog.Debug("ollama.context: extracted context_length", "model", model, "context_length", result.ModelInfo.ContextLength)
+	contextLength := extractContextLength(result.ModelInfo)
+	slog.Debug("ollama.context: extracted context_length", "model", model, "context_length", contextLength)
 
-	if result.ModelInfo.ContextLength <= 0 {
+	if contextLength <= 0 {
 		slog.Debug("ollama.context: context_length not positive, using default", "model", model, "fallback", OllamaDefaultNumCtx)
 		return OllamaDefaultNumCtx
 	}
-	slog.Info("ollama.context: resolved context window", "model", model, "num_ctx", result.ModelInfo.ContextLength)
-	return result.ModelInfo.ContextLength
+	slog.Info("ollama.context: resolved context window", "model", model, "num_ctx", contextLength)
+	return contextLength
+}
+
+// extractContextLength pulls the context window out of an /api/show model_info map.
+// Ollama namespaces the key by model architecture ("gemma4.context_length",
+// "qwen35.context_length", "llama.context_length"), so a fixed "context_length"
+// lookup never matches a real server response; the bare key is still accepted
+// because it is what hand-written fixtures and older stubs return.
+func extractContextLength(modelInfo map[string]json.RawMessage) int {
+	for key, raw := range modelInfo {
+		if key != "context_length" && !strings.HasSuffix(key, ".context_length") {
+			continue
+		}
+		var length int
+		if err := json.Unmarshal(raw, &length); err != nil {
+			continue
+		}
+		if length > 0 {
+			return length
+		}
+	}
+	return 0
 }

internal/providers/ollama_context_test.go

@@ -21,7 +21,14 @@ func TestFetchOllamaModelContext_Success(t *testing.T) {
 
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		require.Equal(t, "/api/show", r.URL.Path)
-		assert.Equal(t, "llama3", r.URL.Query().Get("model"))
+		// /api/show is POST-only and takes the model in a JSON body; a GET with a
+		// query param is answered "405 method not allowed" by a real Ollama server.
+		assert.Equal(t, http.MethodPost, r.Method)
+		var body struct {
+			Model string `json:"model"`
+		}
+		require.NoError(t, json.NewDecoder(r.Body).Decode(&body))
+		assert.Equal(t, "llama3", body.Model)
 
 		resp := response{ModelInfo: modelInfo{ContextLength: 8192}}
 		w.Header().Set("Content-Type", "application/json")
@@ -33,6 +40,21 @@ func TestFetchOllamaModelContext_Success(t *testing.T) {
 	assert.Equal(t, 8192, got)
 }
 
+// TestFetchOllamaModelContext_ArchNamespacedKey covers what a real Ollama server
+// actually returns: the context length is namespaced by model architecture, never
+// exposed under a bare "context_length" key.
+func TestFetchOllamaModelContext_ArchNamespacedKey(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, err := w.Write([]byte(`{"model_info":{"general.architecture":"gemma4","gemma4.context_length":131072,"gemma4.block_count":34}}`))
+		require.NoError(t, err)
+	}))
+	defer srv.Close()
+
+	got := FetchOllamaModelContext(context.Background(), srv.URL, "gemma-4-e4b", "")
+	assert.Equal(t, 131072, got)
+}
+
 func TestFetchOllamaModelContext_SuccessWithV1Suffix(t *testing.T) {
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		assert.Equal(t, "/api/show", r.URL.Path)
@@ -111,3 +133,50 @@ func TestFetchOllamaModelContext_ConnectionRefused(t *testing.T) {
 	got := FetchOllamaModelContext(context.Background(), "http://127.0.0.1:19999", "llama3", "")
 	assert.Equal(t, OllamaDefaultNumCtx, got)
 }
+
+// TestOllamaProviderResolvesNumCtxPerModel proves the provider asks about the model
+// the caller actually requested — not a hardcoded name — and caches the answer.
+func TestOllamaProviderResolvesNumCtxPerModel(t *testing.T) {
+	var calls int
+	var asked []string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		calls++
+		var body struct {
+			Model string `json:"model"`
+		}
+		require.NoError(t, json.NewDecoder(r.Body).Decode(&body))
+		asked = append(asked, body.Model)
+		w.Header().Set("Content-Type", "application/json")
+		_, err := w.Write([]byte(`{"model_info":{"gemma4.context_length":65536}}`))
+		require.NoError(t, err)
+	}))
+	defer srv.Close()
+
+	provider := NewOllamaProvider("test", srv.URL, "unused-default", nil, nil)
+
+	req := ChatRequest{Model: "gemma-4-e4b", Messages: []Message{{Role: "user", Content: "hi"}}}
+	first := provider.buildRequest(context.Background(), req, false)
+	assert.Equal(t, 65536, first.Options["num_ctx"])
+	assert.Equal(t, []string{"gemma-4-e4b"}, asked)
+
+	// Second call for the same model must be served from cache.
+	second := provider.buildRequest(context.Background(), req, false)
+	assert.Equal(t, 65536, second.Options["num_ctx"])
+	assert.Equal(t, 1, calls, "second request for the same model should hit the cache")
+}
+
+// TestOllamaProviderExplicitNumCtxWins proves the operator's configured value takes
+// priority and skips the lookup entirely — the lever for capping KV cache to fit VRAM.
+func TestOllamaProviderExplicitNumCtxWins(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {
+		t.Error("/api/show must not be queried when num_ctx is configured explicitly")
+	}))
+	defer srv.Close()
+
+	configured := 16384
+	provider := NewOllamaProvider("test", srv.URL, "m", &configured, nil)
+
+	req := ChatRequest{Model: "any-model", Messages: []Message{{Role: "user", Content: "hi"}}}
+	built := provider.buildRequest(context.Background(), req, false)
+	assert.Equal(t, 16384, built.Options["num_ctx"])
+}

internal/providers/ollama_test.go

@@ -1,6 +1,7 @@
 package providers
 
 import (
+	"context"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -13,7 +14,7 @@ import (
 func TestOllamaBuildRequest_ThinkDefaultsToFalse(t *testing.T) {
 	p := NewOllamaProvider("ollama", "http://localhost:11434", "llama3.3", nil, nil)
 	req := ChatRequest{Messages: []Message{{Role: "user", Content: "hi"}}}
-	ollamaReq := p.buildRequest(req, false)
+	ollamaReq := p.buildRequest(context.Background(), req, false)
 
 	require.NotNil(t, ollamaReq.Think)
 	value, ok := ollamaReq.Think.Value.(bool)
@@ -41,7 +42,7 @@ func TestOllamaBuildRequest_ThinkOverride(t *testing.T) {
 			p := NewOllamaProvider("ollama", "http://localhost:11434", "llama3.3", nil, nil).
 				WithThinkingEnabled(tc.override)
 			req := ChatRequest{Messages: []Message{{Role: "user", Content: "hi"}}}
-			ollamaReq := p.buildRequest(req, false)
+			ollamaReq := p.buildRequest(context.Background(), req, false)
 
 			require.NotNil(t, ollamaReq.Think)
 			value, ok := ollamaReq.Think.Value.(bool)