diff --git a/cmd/gateway_providers.go b/cmd/gateway_providers.go index 5a3a8d8b..cc587866 100644 --- a/cmd/gateway_providers.go +++ b/cmd/gateway_providers.go @@ -316,7 +316,7 @@ func registerProvidersFromDB(registry *providers.Registry, provStore store.Provi if host == "" { host = "http://localhost:11434" } - numCtx := resolveOllamaNumCtx(&p, config.DockerLocalhost(host), "") + numCtx := resolveOllamaNumCtx(&p) prov := providers.NewOllamaProvider(p.Name, config.DockerLocalhost(host), "llama3.3", numCtx, nil). WithThinkingEnabled(store.ParseThinkingEnabled(p.Settings)) registry.RegisterForTenant(p.TenantID, prov) @@ -397,7 +397,7 @@ func registerProvidersFromDB(registry *providers.Registry, provStore store.Provi if base == "" { base = "https://ollama.com" } - numCtx := resolveOllamaNumCtx(&p, base, p.APIKey) + numCtx := resolveOllamaNumCtx(&p) prov := providers.NewOllamaProvider(p.Name, base, "llama3.3", numCtx, nil). WithThinkingEnabled(store.ParseThinkingEnabled(p.Settings)) registry.RegisterForTenant(p.TenantID, prov) @@ -466,24 +466,18 @@ func openAIProviderDefaults(providerType, apiBase string) (string, string) { } } -// resolveOllamaNumCtx returns the num_ctx to use for an Ollama provider, or nil -// when the built-in default should be used (provider handles it internally). -// Priority: -// 1. User-configured num_ctx from provider settings JSONB (explicit override wins). -// 2. Value queried from Ollama /api/show for the provider's default model. -// 3. nil when neither is available (OllamaProvider omits options.num_ctx, using Ollama's default). -func resolveOllamaNumCtx(p *store.LLMProviderData, apiBase, apiKey string) *int { +// resolveOllamaNumCtx returns the operator-configured num_ctx for an Ollama +// provider, or nil to let the provider resolve it per model at request time. +// +// Only the explicit settings JSONB override is honoured here. Probing /api/show +// at startup cannot work: the model an agent will use is not known until it +// sends a request, so the probe had to guess a model name, and a wrong guess +// resolved to nothing. OllamaProvider.resolveNumCtx does the lookup against the +// real model instead, and caches it. +func resolveOllamaNumCtx(p *store.LLMProviderData) *int { if s := store.ParseOllamaSettings(p.Settings); s != nil { return s.NumCtx } - // Query the Ollama API for the model's native context length. - // Use a short timeout so startup is not blocked by a slow/absent Ollama server. - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - numCtx := providers.FetchOllamaModelContext(ctx, apiBase, "llama3.3", apiKey) - if numCtx != providers.OllamaDefaultNumCtx { - return &numCtx - } return nil } diff --git a/internal/providers/error_classify.go b/internal/providers/error_classify.go index 44ff5bd3..61df90f3 100644 --- a/internal/providers/error_classify.go +++ b/internal/providers/error_classify.go @@ -169,6 +169,7 @@ func isContextOverflow(lower string) bool { "prompt exceeds max length", // ZAI/GLM-5 "request_too_large", // Generic "input is too long", // DashScope + "exceed_context_size", // Ollama native /api/chat 400 "请求输入过长", // Chinese generic ) } diff --git a/internal/providers/error_classify_test.go b/internal/providers/error_classify_test.go index bb9c3ab7..6a941186 100644 --- a/internal/providers/error_classify_test.go +++ b/internal/providers/error_classify_test.go @@ -111,6 +111,14 @@ func TestClassifyContextWindowExceeded(t *testing.T) { } } +func TestClassifyContextWindowOllama(t *testing.T) { + classifier := NewDefaultClassifier() + result := classifier.Classify(nil, 400, `{"error":"exceed_context_size_error"}`) + if result.Kind != "context_overflow" { + t.Errorf("expected context_overflow kind for Ollama, got %s", result.Kind) + } +} + func TestClassifyContextWindowEnglish(t *testing.T) { classifier := NewDefaultClassifier() result := classifier.Classify(nil, 400, "error: maximum context length reached") diff --git a/internal/providers/ollama.go b/internal/providers/ollama.go index c767d072..b484c494 100644 --- a/internal/providers/ollama.go +++ b/internal/providers/ollama.go @@ -7,6 +7,7 @@ import ( "net/http" "net/url" "strings" + "sync" ollamaapi "github.com/ollama/ollama/api" ) @@ -23,6 +24,12 @@ type OllamaProvider struct { client *ollamaapi.Client retryConfig RetryConfig + // ctxCache memoises each model's context window as reported by /api/show, + // keyed by model name. Resolution happens on the first request for a model + // rather than at startup, because the model an agent uses is only known then. + ctxMu sync.RWMutex + ctxCache map[string]int + // thinkingEnabled is the provider-level override for whether requests // should ask Ollama to emit visible reasoning/thinking tokens. // nil = default off (see buildRequest). @@ -59,9 +66,42 @@ func NewOllamaProvider(name, apiBase, defaultModel string, numCtx *int, httpClie numCtx: numCtx, client: ollamaapi.NewClient(parsedURL, httpClient), retryConfig: DefaultRetryConfig(), + ctxCache: make(map[string]int), } } +// resolveNumCtx returns the context window to request for a given model. +// +// An explicitly configured num_ctx always wins — it is the operator's lever for +// capping the KV cache when a model's full window would not fit in VRAM. +// Otherwise the model's own context length is read from /api/show and cached, +// bounded by OllamaDefaultNumCtx so an enormous advertised window (Qwen3.5 +// reports 262144) cannot balloon the KV cache allocation. +// +// Never returns less than the caller would need: on any lookup failure it falls +// back to OllamaDefaultNumCtx rather than to Ollama's 4096 default, which +// silently rejects any prompt larger than a few thousand tokens. +func (p *OllamaProvider) resolveNumCtx(ctx context.Context, model string) int { + if p.numCtx != nil { + return *p.numCtx + } + + p.ctxMu.RLock() + cached, ok := p.ctxCache[model] + p.ctxMu.RUnlock() + if ok { + return cached + } + + numCtx := min(FetchOllamaModelContext(ctx, p.apiBase, model, p.apiKey), OllamaDefaultNumCtx) + + p.ctxMu.Lock() + p.ctxCache[model] = numCtx + p.ctxMu.Unlock() + + return numCtx +} + // WithThinkingEnabled sets the provider-level override for whether native // Ollama chat requests should ask the model to emit visible reasoning // ("think") tokens. nil (not calling this) preserves the existing default @@ -113,7 +153,7 @@ func (p *OllamaProvider) Chat(ctx context.Context, req ChatRequest) (*ChatRespon } return nil } - if err := p.client.Chat(ctx, p.buildRequest(req, false), streamFn); err != nil { + if err := p.client.Chat(ctx, p.buildRequest(ctx, req, false), streamFn); err != nil { return nil, fmt.Errorf("%s: chat: %w", p.name, err) } if finalResp == nil { @@ -171,7 +211,7 @@ func (p *OllamaProvider) ChatStream(ctx context.Context, req ChatRequest, onChun return nil } - if err := p.client.Chat(ctx, p.buildRequest(req, true), streamFn); err != nil { + if err := p.client.Chat(ctx, p.buildRequest(ctx, req, true), streamFn); err != nil { return nil, fmt.Errorf("%s: chat stream: %w", p.name, err) } return acc, nil @@ -183,7 +223,7 @@ func (p *OllamaProvider) ChatStream(ctx context.Context, req ChatRequest, onChun } // buildRequest converts a generic ChatRequest into an Ollama-native api.ChatRequest. -func (p *OllamaProvider) buildRequest(req ChatRequest, stream bool) *ollamaapi.ChatRequest { +func (p *OllamaProvider) buildRequest(ctx context.Context, req ChatRequest, stream bool) *ollamaapi.ChatRequest { model := req.Model if model == "" { model = p.defaultModel @@ -268,13 +308,11 @@ func (p *OllamaProvider) buildRequest(req ChatRequest, stream bool) *ollamaapi.C // Build options map: num_ctx + caller overrides. // Always set num_ctx so Ollama uses a large context window even when the caller // did not configure a specific value. Without this, Ollama defaults to 4096 and - // rejects conversations that exceed that limit. + // rejects conversations that exceed that limit. Note this only works because the + // request goes to the native /api/chat endpoint — the OpenAI-compat shim at + // /v1/chat/completions drops options.num_ctx on the floor. opts := make(map[string]any) - if p.numCtx != nil { - opts["num_ctx"] = *p.numCtx - } else { - opts["num_ctx"] = OllamaDefaultNumCtx - } + opts["num_ctx"] = p.resolveNumCtx(ctx, model) if temp, ok := req.Options[OptTemperature]; ok { opts["temperature"] = temp } diff --git a/internal/providers/ollama_context.go b/internal/providers/ollama_context.go index c2627421..63a49c83 100644 --- a/internal/providers/ollama_context.go +++ b/internal/providers/ollama_context.go @@ -1,6 +1,7 @@ package providers import ( + "bytes" "context" "encoding/json" "fmt" @@ -28,14 +29,19 @@ func FetchOllamaModelContext(ctx context.Context, apiBase, model, apiKey string) slog.Debug("ollama.context: querying /api/show", "api_base", apiBase, "resolved_base", base, "model", model) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + // /api/show is POST-only: a GET is answered with "405 method not allowed", + // which silently degraded every lookup to the fallback. + payload, err := json.Marshal(map[string]string{"model": model}) + if err != nil { + slog.Warn("ollama.context: encode request failed", "model", model, "error", err, "fallback", OllamaDefaultNumCtx) + return OllamaDefaultNumCtx + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload)) if err != nil { slog.Warn("ollama.context: build request failed", "model", model, "error", err, "fallback", OllamaDefaultNumCtx) return OllamaDefaultNumCtx } - q := req.URL.Query() - q.Set("model", model) - req.URL.RawQuery = q.Encode() + req.Header.Set("Content-Type", "application/json") if apiKey != "" { req.Header.Set("Authorization", "Bearer "+apiKey) } @@ -61,21 +67,41 @@ func FetchOllamaModelContext(ctx context.Context, apiBase, model, apiKey string) slog.Debug("ollama.context: /api/show raw response", "model", model, "response", string(rawBody)) var result struct { - ModelInfo struct { - ContextLength int `json:"context_length"` - } `json:"model_info"` + ModelInfo map[string]json.RawMessage `json:"model_info"` } if err := json.Unmarshal(rawBody, &result); err != nil { slog.Warn("ollama.context: decode failed", "model", model, "error", fmt.Sprintf("%v", err), "fallback", OllamaDefaultNumCtx) return OllamaDefaultNumCtx } - slog.Debug("ollama.context: extracted context_length", "model", model, "context_length", result.ModelInfo.ContextLength) + contextLength := extractContextLength(result.ModelInfo) + slog.Debug("ollama.context: extracted context_length", "model", model, "context_length", contextLength) - if result.ModelInfo.ContextLength <= 0 { + if contextLength <= 0 { slog.Debug("ollama.context: context_length not positive, using default", "model", model, "fallback", OllamaDefaultNumCtx) return OllamaDefaultNumCtx } - slog.Info("ollama.context: resolved context window", "model", model, "num_ctx", result.ModelInfo.ContextLength) - return result.ModelInfo.ContextLength + slog.Info("ollama.context: resolved context window", "model", model, "num_ctx", contextLength) + return contextLength +} + +// extractContextLength pulls the context window out of an /api/show model_info map. +// Ollama namespaces the key by model architecture ("gemma4.context_length", +// "qwen35.context_length", "llama.context_length"), so a fixed "context_length" +// lookup never matches a real server response; the bare key is still accepted +// because it is what hand-written fixtures and older stubs return. +func extractContextLength(modelInfo map[string]json.RawMessage) int { + for key, raw := range modelInfo { + if key != "context_length" && !strings.HasSuffix(key, ".context_length") { + continue + } + var length int + if err := json.Unmarshal(raw, &length); err != nil { + continue + } + if length > 0 { + return length + } + } + return 0 } diff --git a/internal/providers/ollama_context_test.go b/internal/providers/ollama_context_test.go index 9ae2ed21..e4a92cec 100644 --- a/internal/providers/ollama_context_test.go +++ b/internal/providers/ollama_context_test.go @@ -21,7 +21,14 @@ func TestFetchOllamaModelContext_Success(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { require.Equal(t, "/api/show", r.URL.Path) - assert.Equal(t, "llama3", r.URL.Query().Get("model")) + // /api/show is POST-only and takes the model in a JSON body; a GET with a + // query param is answered "405 method not allowed" by a real Ollama server. + assert.Equal(t, http.MethodPost, r.Method) + var body struct { + Model string `json:"model"` + } + require.NoError(t, json.NewDecoder(r.Body).Decode(&body)) + assert.Equal(t, "llama3", body.Model) resp := response{ModelInfo: modelInfo{ContextLength: 8192}} w.Header().Set("Content-Type", "application/json") @@ -33,6 +40,21 @@ func TestFetchOllamaModelContext_Success(t *testing.T) { assert.Equal(t, 8192, got) } +// TestFetchOllamaModelContext_ArchNamespacedKey covers what a real Ollama server +// actually returns: the context length is namespaced by model architecture, never +// exposed under a bare "context_length" key. +func TestFetchOllamaModelContext_ArchNamespacedKey(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, err := w.Write([]byte(`{"model_info":{"general.architecture":"gemma4","gemma4.context_length":131072,"gemma4.block_count":34}}`)) + require.NoError(t, err) + })) + defer srv.Close() + + got := FetchOllamaModelContext(context.Background(), srv.URL, "gemma-4-e4b", "") + assert.Equal(t, 131072, got) +} + func TestFetchOllamaModelContext_SuccessWithV1Suffix(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { assert.Equal(t, "/api/show", r.URL.Path) @@ -111,3 +133,50 @@ func TestFetchOllamaModelContext_ConnectionRefused(t *testing.T) { got := FetchOllamaModelContext(context.Background(), "http://127.0.0.1:19999", "llama3", "") assert.Equal(t, OllamaDefaultNumCtx, got) } + +// TestOllamaProviderResolvesNumCtxPerModel proves the provider asks about the model +// the caller actually requested — not a hardcoded name — and caches the answer. +func TestOllamaProviderResolvesNumCtxPerModel(t *testing.T) { + var calls int + var asked []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + var body struct { + Model string `json:"model"` + } + require.NoError(t, json.NewDecoder(r.Body).Decode(&body)) + asked = append(asked, body.Model) + w.Header().Set("Content-Type", "application/json") + _, err := w.Write([]byte(`{"model_info":{"gemma4.context_length":65536}}`)) + require.NoError(t, err) + })) + defer srv.Close() + + provider := NewOllamaProvider("test", srv.URL, "unused-default", nil, nil) + + req := ChatRequest{Model: "gemma-4-e4b", Messages: []Message{{Role: "user", Content: "hi"}}} + first := provider.buildRequest(context.Background(), req, false) + assert.Equal(t, 65536, first.Options["num_ctx"]) + assert.Equal(t, []string{"gemma-4-e4b"}, asked) + + // Second call for the same model must be served from cache. + second := provider.buildRequest(context.Background(), req, false) + assert.Equal(t, 65536, second.Options["num_ctx"]) + assert.Equal(t, 1, calls, "second request for the same model should hit the cache") +} + +// TestOllamaProviderExplicitNumCtxWins proves the operator's configured value takes +// priority and skips the lookup entirely — the lever for capping KV cache to fit VRAM. +func TestOllamaProviderExplicitNumCtxWins(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { + t.Error("/api/show must not be queried when num_ctx is configured explicitly") + })) + defer srv.Close() + + configured := 16384 + provider := NewOllamaProvider("test", srv.URL, "m", &configured, nil) + + req := ChatRequest{Model: "any-model", Messages: []Message{{Role: "user", Content: "hi"}}} + built := provider.buildRequest(context.Background(), req, false) + assert.Equal(t, 16384, built.Options["num_ctx"]) +} diff --git a/internal/providers/ollama_test.go b/internal/providers/ollama_test.go index d4e6413a..7eb22356 100644 --- a/internal/providers/ollama_test.go +++ b/internal/providers/ollama_test.go @@ -1,6 +1,7 @@ package providers import ( + "context" "testing" "github.com/stretchr/testify/assert" @@ -13,7 +14,7 @@ import ( func TestOllamaBuildRequest_ThinkDefaultsToFalse(t *testing.T) { p := NewOllamaProvider("ollama", "http://localhost:11434", "llama3.3", nil, nil) req := ChatRequest{Messages: []Message{{Role: "user", Content: "hi"}}} - ollamaReq := p.buildRequest(req, false) + ollamaReq := p.buildRequest(context.Background(), req, false) require.NotNil(t, ollamaReq.Think) value, ok := ollamaReq.Think.Value.(bool) @@ -41,7 +42,7 @@ func TestOllamaBuildRequest_ThinkOverride(t *testing.T) { p := NewOllamaProvider("ollama", "http://localhost:11434", "llama3.3", nil, nil). WithThinkingEnabled(tc.override) req := ChatRequest{Messages: []Message{{Role: "user", Content: "hi"}}} - ollamaReq := p.buildRequest(req, false) + ollamaReq := p.buildRequest(context.Background(), req, false) require.NotNil(t, ollamaReq.Think) value, ok := ollamaReq.Think.Value.(bool)