From 01d75ac7fe699fa1eff4474747b38eb997e7c8de Mon Sep 17 00:00:00 2001 From: viettranx Date: Sun, 8 Mar 2026 20:10:10 +0700 Subject: [PATCH] refactor(tools): migrate read_* tools to provider chain and add media models Refactor read_image, read_document, read_video, read_audio to use ResolveMediaProviderChain + ExecuteWithChain for consistent fallback behavior. Add hardcoded model lists for MiniMax, DashScope, and Suno providers. --- internal/http/provider_models.go | 42 ++++++++ internal/tools/read_audio.go | 61 +++++------ internal/tools/read_audio_resolve.go | 133 +++++------------------- internal/tools/read_document.go | 60 ++++++----- internal/tools/read_document_resolve.go | 122 ++++------------------ internal/tools/read_image.go | 131 +++++++++-------------- internal/tools/read_video.go | 56 +++++----- internal/tools/read_video_resolve.go | 115 ++++---------------- 8 files changed, 240 insertions(+), 480 deletions(-) diff --git a/internal/http/provider_models.go b/internal/http/provider_models.go index 9e3ad66b..81e88533 100644 --- a/internal/http/provider_models.go +++ b/internal/http/provider_models.go @@ -60,8 +60,12 @@ func (h *ProvidersHandler) handleListProviderModels(w http.ResponseWriter, r *ht models, err = fetchGeminiModels(ctx, p.APIKey) case "bailian": models = bailianModels() + case "dashscope": + models = dashScopeModels() case "minimax_native": models = minimaxModels() + case "suno": + models = sunoModels() default: // All other types use OpenAI-compatible /models endpoint apiBase := strings.TrimRight(p.APIBase, "/") @@ -176,9 +180,47 @@ func bailianModels() []ModelInfo { // MiniMax does not expose a /v1/models endpoint. func minimaxModels() []ModelInfo { return []ModelInfo{ + // Chat / text {ID: "MiniMax-Text-01", Name: "MiniMax Text 01"}, {ID: "MiniMax-M1", Name: "MiniMax M1"}, {ID: "MiniMax-M2.5", Name: "MiniMax M2.5"}, + // Image generation + {ID: "image-01", Name: "Image 01"}, + // Video generation + {ID: "MiniMax-Hailuo-2.3", Name: "Hailuo Video 2.3"}, + {ID: "MiniMax-Hailuo-2", Name: "Hailuo Video 2"}, + {ID: "T2V-01-Director", Name: "T2V-01 Director"}, + // Music generation + {ID: "music-2.5+", Name: "Music 2.5+"}, + {ID: "music-2.5", Name: "Music 2.5"}, + // TTS + {ID: "speech-02-hd", Name: "Speech 02 HD"}, + {ID: "speech-02-turbo", Name: "Speech 02 Turbo"}, + } +} + +// dashScopeModels returns a hardcoded list of DashScope (Qwen) models. +// DashScope does not expose a standard /v1/models endpoint. +func dashScopeModels() []ModelInfo { + return []ModelInfo{ + // Chat / text + {ID: "qwen3-max", Name: "Qwen 3 Max"}, + {ID: "qwen3-plus", Name: "Qwen 3 Plus"}, + {ID: "qwen3-turbo", Name: "Qwen 3 Turbo"}, + // Image generation + {ID: "wan2.6-image", Name: "Wan 2.6 Image"}, + {ID: "wan2.1-image", Name: "Wan 2.1 Image"}, + // Video generation + {ID: "wan2.6-video", Name: "Wan 2.6 Video"}, + } +} + +// sunoModels returns a hardcoded list of Suno music generation models. +func sunoModels() []ModelInfo { + return []ModelInfo{ + {ID: "v4.5", Name: "Suno V4.5"}, + {ID: "v4", Name: "Suno V4"}, + {ID: "v3.5", Name: "Suno V3.5"}, } } diff --git a/internal/tools/read_audio.go b/internal/tools/read_audio.go index f9439118..285aed08 100644 --- a/internal/tools/read_audio.go +++ b/internal/tools/read_audio.go @@ -30,8 +30,18 @@ func MediaAudioRefsFromCtx(ctx context.Context) []providers.MediaRef { // audioMaxBytes is the max file size for audio analysis (50MB). const audioMaxBytes = 50 * 1024 * 1024 +// audioProviderPriority is the order in which providers are tried for audio analysis. +var audioProviderPriority = []string{"gemini", "openai", "openrouter"} + +// audioModelDefaults maps provider names to preferred audio-capable models. +var audioModelDefaults = map[string]string{ + "gemini": "gemini-2.5-flash", + "openai": "gpt-4o-audio-preview", + "openrouter": "google/gemini-2.5-flash", +} + // ReadAudioTool uses an audio-capable provider to analyze audio files -// attached to the current conversation. Follows same pattern as ReadDocumentTool. +// attached to the current conversation. type ReadAudioTool struct { registry *providers.Registry mediaLoader MediaPathLoader @@ -73,7 +83,6 @@ func (t *ReadAudioTool) Execute(ctx context.Context, args map[string]interface{} } mediaID, _ := args["media_id"].(string) - // Resolve audio file path from MediaRefs in context. audioPath, audioMime, err := t.resolveAudioFile(ctx, mediaID) if err != nil { return ErrorResult(err.Error()) @@ -81,7 +90,6 @@ func (t *ReadAudioTool) Execute(ctx context.Context, args map[string]interface{} slog.Info("read_audio: resolved file", "path", audioPath, "mime", audioMime, "media_id", mediaID) - // Read audio file. data, err := os.ReadFile(audioPath) if err != nil { return ErrorResult(fmt.Sprintf("Failed to read audio file: %v", err)) @@ -91,40 +99,27 @@ func (t *ReadAudioTool) Execute(ctx context.Context, args map[string]interface{} return ErrorResult(fmt.Sprintf("Audio too large: %d bytes (max %d)", len(data), audioMaxBytes)) } - // Find an audio-capable provider. - provider, model, err := t.resolveAudioProviderWithConfig(ctx) - if err != nil { - return ErrorResult(err.Error()) - } + chain := ResolveMediaProviderChain(ctx, "read_audio", "", "", + audioProviderPriority, audioModelDefaults, t.registry) - // Try primary provider, fallback to next available on error. - resp, usedProvider, usedModel := t.callAudioProvider(ctx, provider, model, prompt, data, audioMime) - if resp == nil { - // Primary failed — try fallback providers from priority list. - slog.Warn("read_audio: primary provider failed, trying fallback", "primary", provider.Name()) - for _, fbName := range audioProviderPriority { - if fbName == provider.Name() { - continue - } - fbProvider, fbModel, fbErr := t.resolveAudioProviderByName(fbName) - if fbErr != nil { - continue - } - resp, usedProvider, usedModel = t.callAudioProvider(ctx, fbProvider, fbModel, prompt, data, audioMime) - if resp != nil { - slog.Info("read_audio: fallback succeeded", "provider", usedProvider) - break - } + for i := range chain { + if chain[i].Params == nil { + chain[i].Params = make(map[string]any) } - } - if resp == nil { - return ErrorResult("Audio analysis failed: all providers returned errors") + chain[i].Params["prompt"] = prompt + chain[i].Params["data"] = data + chain[i].Params["mime"] = audioMime } - result := NewResult(resp.Content) - result.Usage = resp.Usage - result.Provider = usedProvider - result.Model = usedModel + chainResult, err := ExecuteWithChain(ctx, chain, t.registry, t.callProvider) + if err != nil { + return ErrorResult(fmt.Sprintf("Audio analysis failed: %v", err)) + } + + result := NewResult(string(chainResult.Data)) + result.Usage = chainResult.Usage + result.Provider = chainResult.Provider + result.Model = chainResult.Model return result } diff --git a/internal/tools/read_audio_resolve.go b/internal/tools/read_audio_resolve.go index 2da82ea0..ac3e1895 100644 --- a/internal/tools/read_audio_resolve.go +++ b/internal/tools/read_audio_resolve.go @@ -3,7 +3,6 @@ package tools import ( "context" "encoding/base64" - "encoding/json" "fmt" "log/slog" "path/filepath" @@ -13,16 +12,6 @@ import ( "github.com/nextlevelbuilder/goclaw/internal/providers" ) -// audioProviderPriority is the order in which providers are tried for audio analysis. -var audioProviderPriority = []string{"gemini", "openai", "openrouter"} - -// audioModelOverrides maps provider names to preferred audio-capable models. -var audioModelOverrides = map[string]string{ - "gemini": "gemini-2.5-flash", - "openai": "gpt-4o-audio-preview", - "openrouter": "google/gemini-2.5-flash", -} - // resolveAudioFile finds the audio file path from context MediaRefs. func (t *ReadAudioTool) resolveAudioFile(ctx context.Context, mediaID string) (path, mime string, err error) { if t.mediaLoader == nil { @@ -62,114 +51,43 @@ func (t *ReadAudioTool) resolveAudioFile(ctx context.Context, mediaID string) (p return p, mime, nil } -// resolveAudioProviderWithConfig checks builtin settings, then hardcoded priority. -func (t *ReadAudioTool) resolveAudioProviderWithConfig(ctx context.Context) (providers.Provider, string, error) { - if p, model, ok := t.resolveFromAudioSettings(ctx); ok { - return p, model, nil - } - return t.resolveAudioProvider() -} - -// resolveFromAudioSettings checks global builtin tool settings for provider/model config. -func (t *ReadAudioTool) resolveFromAudioSettings(ctx context.Context) (providers.Provider, string, bool) { - settings := BuiltinToolSettingsFromCtx(ctx) - if settings == nil { - return nil, "", false - } - raw, ok := settings["read_audio"] - if !ok || len(raw) == 0 { - return nil, "", false - } - var cfg struct { - Provider string `json:"provider"` - Model string `json:"model"` - } - if err := json.Unmarshal(raw, &cfg); err != nil || cfg.Provider == "" { - return nil, "", false - } - p, err := t.registry.Get(cfg.Provider) - if err != nil { - return nil, "", false - } - model := cfg.Model - if model == "" { - model = p.DefaultModel() - } - return p, model, true -} - -// resolveAudioProvider finds the first available audio-capable provider. -func (t *ReadAudioTool) resolveAudioProvider() (providers.Provider, string, error) { - for _, name := range audioProviderPriority { - p, err := t.registry.Get(name) - if err != nil { - continue - } - model := p.DefaultModel() - if override, ok := audioModelOverrides[name]; ok { - model = override - } - return p, model, nil - } - return nil, "", fmt.Errorf("no audio-capable provider available (need one of: %v)", audioProviderPriority) -} - -// resolveAudioProviderByName gets a specific provider by name and applies model override. -func (t *ReadAudioTool) resolveAudioProviderByName(name string) (providers.Provider, string, error) { - p, err := t.registry.Get(name) - if err != nil { - return nil, "", err - } - model := p.DefaultModel() - if override, ok := audioModelOverrides[name]; ok { - model = override - } - return p, model, nil -} - -// callAudioProvider sends audio to a provider for analysis. +// callProvider dispatches audio analysis to the appropriate provider API. // Gemini: uses File API (upload → poll → file_data in generateContent). // OpenAI: uses input_audio content part in chat completions. -// Others: falls back to base64 in image_url (may not work for all). -func (t *ReadAudioTool) callAudioProvider(ctx context.Context, provider providers.Provider, model, prompt string, data []byte, mime string) (*providers.ChatResponse, string, string) { - provName := provider.Name() +// Others: falls back to base64 in image_url (best effort). +func (t *ReadAudioTool) callProvider(ctx context.Context, cp credentialProvider, providerName, model string, params map[string]any) ([]byte, *providers.Usage, error) { + prompt := GetParamString(params, "prompt", "Analyze this audio and describe its contents.") + data, _ := params["data"].([]byte) + mime := GetParamString(params, "mime", "audio/mpeg") // Gemini: use File API (inlineData doesn't work for audio). - if strings.HasPrefix(provName, "gemini") { - oaiProv, ok := provider.(*providers.OpenAIProvider) - if !ok { - slog.Warn("read_audio: gemini provider is not OpenAIProvider", "provider", provName) - return nil, "", "" - } - apiKey := oaiProv.APIKey() - slog.Info("read_audio: using gemini file API", "provider", provName, "model", model, "size", len(data), "mime", mime) - resp, err := geminiFileAPICall(ctx, apiKey, model, prompt, data, mime, 120*time.Second) + if strings.HasPrefix(providerName, "gemini") { + slog.Info("read_audio: using gemini file API", "provider", providerName, "model", model, "size", len(data), "mime", mime) + resp, err := geminiFileAPICall(ctx, cp.APIKey(), model, prompt, data, mime, 120*time.Second) if err != nil { - slog.Warn("read_audio: gemini file API call failed", "error", err) - return nil, "", "" + return nil, nil, fmt.Errorf("gemini file API: %w", err) } - return resp, provName, model + return []byte(resp.Content), resp.Usage, nil } // OpenAI: use input_audio content part (supports wav, mp3). - if strings.HasPrefix(provName, "openai") { - oaiProv, ok := provider.(*providers.OpenAIProvider) - if !ok { - slog.Warn("read_audio: openai provider is not OpenAIProvider", "provider", provName) - return nil, "", "" - } - slog.Info("read_audio: using openai input_audio API", "provider", provName, "model", model, "size", len(data), "mime", mime) - resp, err := openaiAudioCall(ctx, oaiProv.APIKey(), oaiProv.APIBase(), model, prompt, data, mime) + if strings.HasPrefix(providerName, "openai") { + slog.Info("read_audio: using openai input_audio API", "provider", providerName, "model", model, "size", len(data), "mime", mime) + resp, err := openaiAudioCall(ctx, cp.APIKey(), cp.APIBase(), model, prompt, data, mime) if err != nil { - slog.Warn("read_audio: openai audio call failed", "error", err) - return nil, "", "" + return nil, nil, fmt.Errorf("openai audio call: %w", err) } - return resp, provName, model + return []byte(resp.Content), resp.Usage, nil } // Other providers: try standard Chat API with base64 audio as image_url (best effort). - slog.Info("read_audio: using chat API fallback", "provider", provName, "model", model, "size", len(data)) - resp, err := provider.Chat(ctx, providers.ChatRequest{ + p, err := t.registry.Get(providerName) + if err != nil { + return nil, nil, fmt.Errorf("provider %q not available: %w", providerName, err) + } + + slog.Info("read_audio: using chat API fallback", "provider", providerName, "model", model, "size", len(data)) + resp, err := p.Chat(ctx, providers.ChatRequest{ Messages: []providers.Message{ { Role: "user", @@ -184,10 +102,9 @@ func (t *ReadAudioTool) callAudioProvider(ctx context.Context, provider provider }, }) if err != nil { - slog.Warn("read_audio: chat API fallback failed", "provider", provName, "error", err) - return nil, "", "" + return nil, nil, fmt.Errorf("chat API: %w", err) } - return resp, provName, model + return []byte(resp.Content), resp.Usage, nil } // openaiAudioCall sends audio to OpenAI using the input_audio content part. diff --git a/internal/tools/read_document.go b/internal/tools/read_document.go index 980819a9..d3bbfc17 100644 --- a/internal/tools/read_document.go +++ b/internal/tools/read_document.go @@ -48,6 +48,17 @@ func MediaDocRefsFromCtx(ctx context.Context) []providers.MediaRef { // documentMaxBytes is the max file size for document analysis (20MB). const documentMaxBytes = 20 * 1024 * 1024 +// documentProviderPriority is the order in which providers are tried for document analysis. +// Gemini has best native PDF support (50MB, 258 tokens/page). +var documentProviderPriority = []string{"gemini", "anthropic", "openrouter", "dashscope"} + +// documentModelDefaults maps provider names to preferred document-capable models. +var documentModelDefaults = map[string]string{ + "gemini": "gemini-2.5-flash", + "openrouter": "google/gemini-2.5-flash", + "dashscope": "qwen-vl-max", +} + // ReadDocumentTool uses a document-capable provider to analyze files // attached to the current conversation. Follows same pattern as ReadImageTool. type ReadDocumentTool struct { @@ -119,40 +130,27 @@ func (t *ReadDocumentTool) Execute(ctx context.Context, args map[string]interfac return NewResult(content) } - // Find a document-capable provider. - provider, model, err := t.resolveDocumentProviderWithConfig(ctx) - if err != nil { - return ErrorResult(err.Error()) - } + chain := ResolveMediaProviderChain(ctx, "read_document", "", "", + documentProviderPriority, documentModelDefaults, t.registry) - // Try primary provider, fallback to next available on error. - resp, usedProvider, usedModel := t.callDocumentProvider(ctx, provider, model, prompt, data, docMime) - if resp == nil { - // Primary failed — try fallback providers from priority list. - slog.Warn("read_document: primary provider failed, trying fallback", "primary", provider.Name()) - for _, fbName := range documentProviderPriority { - if fbName == provider.Name() { - continue // skip the one that already failed - } - fbProvider, fbModel, fbErr := t.resolveDocumentProviderByName(fbName) - if fbErr != nil { - continue - } - resp, usedProvider, usedModel = t.callDocumentProvider(ctx, fbProvider, fbModel, prompt, data, docMime) - if resp != nil { - slog.Info("read_document: fallback succeeded", "provider", usedProvider) - break - } + // Inject prompt, data, and mime into each chain entry's params + for i := range chain { + if chain[i].Params == nil { + chain[i].Params = make(map[string]any) } - } - if resp == nil { - return ErrorResult("Document analysis failed: all providers returned errors") + chain[i].Params["prompt"] = prompt + chain[i].Params["data"] = data + chain[i].Params["mime"] = docMime } - result := NewResult(resp.Content) - result.Usage = resp.Usage - result.Provider = usedProvider - result.Model = usedModel + chainResult, err := ExecuteWithChain(ctx, chain, t.registry, t.callProvider) + if err != nil { + return ErrorResult(fmt.Sprintf("Document analysis failed: %v", err)) + } + + result := NewResult(string(chainResult.Data)) + result.Usage = chainResult.Usage + result.Provider = chainResult.Provider + result.Model = chainResult.Model return result } - diff --git a/internal/tools/read_document_resolve.go b/internal/tools/read_document_resolve.go index 53dab51e..60dd9b80 100644 --- a/internal/tools/read_document_resolve.go +++ b/internal/tools/read_document_resolve.go @@ -3,7 +3,6 @@ package tools import ( "context" "encoding/base64" - "encoding/json" "fmt" "log/slog" "path/filepath" @@ -12,17 +11,6 @@ import ( "github.com/nextlevelbuilder/goclaw/internal/providers" ) -// documentProviderPriority is the order in which providers are tried for document analysis. -// Gemini has best native PDF support (50MB, 258 tokens/page). -var documentProviderPriority = []string{"gemini", "anthropic", "openrouter", "dashscope"} - -// documentModelOverrides maps provider names to preferred document-capable models. -var documentModelOverrides = map[string]string{ - "gemini": "gemini-2.5-flash", - "openrouter": "google/gemini-2.5-flash", - "dashscope": "qwen-vl-max", -} - // resolveDocumentFile finds the document file path from context MediaRefs. func (t *ReadDocumentTool) resolveDocumentFile(ctx context.Context, mediaID string) (path, mime string, err error) { if t.mediaLoader == nil { @@ -65,88 +53,34 @@ func (t *ReadDocumentTool) resolveDocumentFile(ctx context.Context, mediaID stri return p, mime, nil } -// resolveDocumentProviderWithConfig checks per-agent config, global settings, then hardcoded priority. -func (t *ReadDocumentTool) resolveDocumentProviderWithConfig(ctx context.Context) (providers.Provider, string, error) { - // 1. Global builtin_tools.settings - if p, model, ok := t.resolveFromBuiltinSettings(ctx); ok { - return p, model, nil - } - // 2. Hardcoded defaults - return t.resolveDocumentProvider() -} - -// resolveFromBuiltinSettings checks global builtin tool settings for provider/model config. -func (t *ReadDocumentTool) resolveFromBuiltinSettings(ctx context.Context) (providers.Provider, string, bool) { - settings := BuiltinToolSettingsFromCtx(ctx) - if settings == nil { - return nil, "", false - } - raw, ok := settings["read_document"] - if !ok || len(raw) == 0 { - return nil, "", false - } - var cfg struct { - Provider string `json:"provider"` - Model string `json:"model"` - } - if err := json.Unmarshal(raw, &cfg); err != nil || cfg.Provider == "" { - return nil, "", false - } - p, err := t.registry.Get(cfg.Provider) - if err != nil { - return nil, "", false - } - model := cfg.Model - if model == "" { - model = p.DefaultModel() - } - return p, model, true -} - -// resolveDocumentProvider finds the first available document-capable provider. -func (t *ReadDocumentTool) resolveDocumentProvider() (providers.Provider, string, error) { - for _, name := range documentProviderPriority { - p, err := t.registry.Get(name) - if err != nil { - continue - } - model := p.DefaultModel() - if override, ok := documentModelOverrides[name]; ok { - model = override - } - return p, model, nil - } - return nil, "", fmt.Errorf("no document-capable provider available (need one of: %v)", documentProviderPriority) -} - -// callDocumentProvider sends a document to a provider for analysis. -// For Gemini providers, uses native generateContent API (supports PDF natively). -// For others, falls back to OpenAI-compat chat with base64 document. -func (t *ReadDocumentTool) callDocumentProvider(ctx context.Context, provider providers.Provider, model, prompt string, data []byte, mime string) (*providers.ChatResponse, string, string) { - provName := provider.Name() +// callProvider dispatches document analysis to the appropriate provider API. +// For Gemini: uses native generateContent API (supports PDF natively). +// For others: uses standard Chat API with base64 document. +func (t *ReadDocumentTool) callProvider(ctx context.Context, cp credentialProvider, providerName, model string, params map[string]any) ([]byte, *providers.Usage, error) { + prompt := GetParamString(params, "prompt", "Analyze this document and describe its contents.") + data, _ := params["data"].([]byte) + mime := GetParamString(params, "mime", "application/octet-stream") // Gemini: use native API (OpenAI-compat endpoint doesn't support non-image MIME types). - if strings.HasPrefix(provName, "gemini") { - oaiProv, ok := provider.(*providers.OpenAIProvider) - if !ok { - slog.Warn("read_document: gemini provider is not OpenAIProvider", "provider", provName) - return nil, "", "" - } - apiKey := oaiProv.APIKey() + if strings.HasPrefix(providerName, "gemini") { slog.Info("read_document: using gemini native API", - "provider", provName, "model", model, + "provider", providerName, "model", model, "doc_size", len(data), "mime", mime) - resp, err := geminiNativeDocumentCall(ctx, apiKey, model, prompt, data, mime) + resp, err := geminiNativeDocumentCall(ctx, cp.APIKey(), model, prompt, data, mime) if err != nil { - slog.Warn("read_document: gemini native call failed", "error", err) - return nil, "", "" + return nil, nil, fmt.Errorf("gemini native call: %w", err) } - return resp, provName, model + return []byte(resp.Content), resp.Usage, nil } // Other providers: use standard Chat API with document as base64 image_url. - slog.Info("read_document: using chat API", "provider", provName, "model", model, "doc_size", len(data)) - resp, err := provider.Chat(ctx, providers.ChatRequest{ + p, err := t.registry.Get(providerName) + if err != nil { + return nil, nil, fmt.Errorf("provider %q not available: %w", providerName, err) + } + + slog.Info("read_document: using chat API", "provider", providerName, "model", model, "doc_size", len(data)) + resp, err := p.Chat(ctx, providers.ChatRequest{ Messages: []providers.Message{ { Role: "user", @@ -161,23 +95,9 @@ func (t *ReadDocumentTool) callDocumentProvider(ctx context.Context, provider pr }, }) if err != nil { - slog.Warn("read_document: chat call failed", "provider", provName, "error", err) - return nil, "", "" + return nil, nil, fmt.Errorf("chat call: %w", err) } - return resp, provName, model -} - -// resolveDocumentProviderByName gets a specific provider by name and applies model override. -func (t *ReadDocumentTool) resolveDocumentProviderByName(name string) (providers.Provider, string, error) { - p, err := t.registry.Get(name) - if err != nil { - return nil, "", err - } - model := p.DefaultModel() - if override, ok := documentModelOverrides[name]; ok { - model = override - } - return p, model, nil + return []byte(resp.Content), resp.Usage, nil } // mimeFromDocExt returns MIME type for document file extensions. diff --git a/internal/tools/read_image.go b/internal/tools/read_image.go index 3126410f..712c8758 100644 --- a/internal/tools/read_image.go +++ b/internal/tools/read_image.go @@ -2,7 +2,6 @@ package tools import ( "context" - "encoding/json" "fmt" "log/slog" @@ -29,10 +28,11 @@ func MediaImagesFromCtx(ctx context.Context) []providers.ImageContent { // visionProviderPriority is the order in which providers are tried for vision. var visionProviderPriority = []string{"openrouter", "gemini", "anthropic", "dashscope"} -// visionModelOverrides maps provider names to preferred vision models. -// Providers not listed here use their default model. -var visionModelOverrides = map[string]string{ +// visionModelDefaults maps provider names to preferred vision models. +var visionModelDefaults = map[string]string{ "openrouter": "google/gemini-2.5-flash-image", + "gemini": "gemini-2.5-flash", + "anthropic": "", "dashscope": "qwen3-vl", } @@ -75,15 +75,51 @@ func (t *ReadImageTool) Execute(ctx context.Context, args map[string]interface{} return ErrorResult("No images available in this conversation. The user may not have sent an image.") } - // Find a vision-capable provider (per-agent config > hardcoded priority) - provider, model, err := t.resolveVisionProviderWithConfig(ctx) - if err != nil { - return ErrorResult(err.Error()) + // Extract per-agent config for backward compat + var perAgentProvider, perAgentModel string + if cfg := VisionConfigFromCtx(ctx); cfg != nil { + perAgentProvider = cfg.Provider + perAgentModel = cfg.Model } - slog.Info("read_image: calling vision provider", "provider", provider.Name(), "model", model, "images", len(images)) + chain := ResolveMediaProviderChain(ctx, "read_image", perAgentProvider, perAgentModel, + visionProviderPriority, visionModelDefaults, t.registry) - resp, err := provider.Chat(ctx, providers.ChatRequest{ + // Inject prompt and images into each chain entry's params + for i := range chain { + if chain[i].Params == nil { + chain[i].Params = make(map[string]any) + } + chain[i].Params["prompt"] = prompt + chain[i].Params["images"] = images + } + + chainResult, err := ExecuteWithChain(ctx, chain, t.registry, t.callProvider) + if err != nil { + return ErrorResult(fmt.Sprintf("image analysis failed: %v", err)) + } + + result := NewResult(string(chainResult.Data)) + result.Usage = chainResult.Usage + result.Provider = chainResult.Provider + result.Model = chainResult.Model + return result +} + +// callProvider dispatches the vision call using provider.Chat(). +func (t *ReadImageTool) callProvider(ctx context.Context, cp credentialProvider, providerName, model string, params map[string]any) ([]byte, *providers.Usage, error) { + prompt := GetParamString(params, "prompt", "Describe this image in detail.") + images, _ := params["images"].([]providers.ImageContent) + + // Get the full provider for Chat() access + p, err := t.registry.Get(providerName) + if err != nil { + return nil, nil, fmt.Errorf("provider %q not available: %w", providerName, err) + } + + slog.Info("read_image: calling vision provider", "provider", providerName, "model", model, "images", len(images)) + + resp, err := p.Chat(ctx, providers.ChatRequest{ Messages: []providers.Message{ { Role: "user", @@ -98,79 +134,8 @@ func (t *ReadImageTool) Execute(ctx context.Context, args map[string]interface{} }, }) if err != nil { - return ErrorResult(fmt.Sprintf("Vision provider error: %v", err)) + return nil, nil, fmt.Errorf("vision provider error: %w", err) } - result := NewResult(resp.Content) - result.Usage = resp.Usage - result.Provider = provider.Name() - result.Model = model - return result -} - -// resolveVisionProviderWithConfig checks per-agent VisionConfig first, -// then global builtin_tools.settings, then falls back to hardcoded priority. -func (t *ReadImageTool) resolveVisionProviderWithConfig(ctx context.Context) (providers.Provider, string, error) { - // 1. Per-agent override (highest priority) - if cfg := VisionConfigFromCtx(ctx); cfg != nil && cfg.Provider != "" { - p, err := t.registry.Get(cfg.Provider) - if err != nil { - return nil, "", fmt.Errorf("configured vision provider %q not available: %w", cfg.Provider, err) - } - model := cfg.Model - if model == "" { - model = p.DefaultModel() - } - return p, model, nil - } - // 2. Global builtin_tools.settings (DB defaults) - if p, model, ok := t.resolveFromBuiltinSettings(ctx); ok { - return p, model, nil - } - // 3. Hardcoded defaults - return t.resolveVisionProvider() -} - -// resolveFromBuiltinSettings checks global builtin tool settings for provider/model config. -func (t *ReadImageTool) resolveFromBuiltinSettings(ctx context.Context) (providers.Provider, string, bool) { - settings := BuiltinToolSettingsFromCtx(ctx) - if settings == nil { - return nil, "", false - } - raw, ok := settings["read_image"] - if !ok || len(raw) == 0 { - return nil, "", false - } - var cfg struct { - Provider string `json:"provider"` - Model string `json:"model"` - } - if err := json.Unmarshal(raw, &cfg); err != nil || cfg.Provider == "" { - return nil, "", false - } - p, err := t.registry.Get(cfg.Provider) - if err != nil { - return nil, "", false - } - model := cfg.Model - if model == "" { - model = p.DefaultModel() - } - return p, model, true -} - -// resolveVisionProvider finds the first available vision-capable provider. -func (t *ReadImageTool) resolveVisionProvider() (providers.Provider, string, error) { - for _, name := range visionProviderPriority { - p, err := t.registry.Get(name) - if err != nil { - continue - } - model := p.DefaultModel() - if override, ok := visionModelOverrides[name]; ok { - model = override - } - return p, model, nil - } - return nil, "", fmt.Errorf("no vision-capable provider available (need one of: %v)", visionProviderPriority) + return []byte(resp.Content), resp.Usage, nil } diff --git a/internal/tools/read_video.go b/internal/tools/read_video.go index 5ee9f6dd..23f6ea8a 100644 --- a/internal/tools/read_video.go +++ b/internal/tools/read_video.go @@ -30,8 +30,18 @@ func MediaVideoRefsFromCtx(ctx context.Context) []providers.MediaRef { // videoMaxBytes is the max file size for video analysis (100MB). const videoMaxBytes = 100 * 1024 * 1024 +// videoProviderPriority is the order in which providers are tried for video analysis. +// OpenAI excluded — no native video upload in chat completions. +var videoProviderPriority = []string{"gemini", "openrouter"} + +// videoModelDefaults maps provider names to preferred video-capable models. +var videoModelDefaults = map[string]string{ + "gemini": "gemini-2.5-flash", + "openrouter": "google/gemini-2.5-flash", +} + // ReadVideoTool uses a video-capable provider to analyze video files -// attached to the current conversation. Follows same pattern as ReadAudioTool. +// attached to the current conversation. type ReadVideoTool struct { registry *providers.Registry mediaLoader MediaPathLoader @@ -89,37 +99,27 @@ func (t *ReadVideoTool) Execute(ctx context.Context, args map[string]interface{} return ErrorResult(fmt.Sprintf("Video too large: %d bytes (max %d)", len(data), videoMaxBytes)) } - provider, model, err := t.resolveVideoProviderWithConfig(ctx) - if err != nil { - return ErrorResult(err.Error()) - } + chain := ResolveMediaProviderChain(ctx, "read_video", "", "", + videoProviderPriority, videoModelDefaults, t.registry) - resp, usedProvider, usedModel := t.callVideoProvider(ctx, provider, model, prompt, data, videoMime) - if resp == nil { - slog.Warn("read_video: primary provider failed, trying fallback", "primary", provider.Name()) - for _, fbName := range videoProviderPriority { - if fbName == provider.Name() { - continue - } - fbProvider, fbModel, fbErr := t.resolveVideoProviderByName(fbName) - if fbErr != nil { - continue - } - resp, usedProvider, usedModel = t.callVideoProvider(ctx, fbProvider, fbModel, prompt, data, videoMime) - if resp != nil { - slog.Info("read_video: fallback succeeded", "provider", usedProvider) - break - } + for i := range chain { + if chain[i].Params == nil { + chain[i].Params = make(map[string]any) } - } - if resp == nil { - return ErrorResult("Video analysis failed: all providers returned errors") + chain[i].Params["prompt"] = prompt + chain[i].Params["data"] = data + chain[i].Params["mime"] = videoMime } - result := NewResult(resp.Content) - result.Usage = resp.Usage - result.Provider = usedProvider - result.Model = usedModel + chainResult, err := ExecuteWithChain(ctx, chain, t.registry, t.callProvider) + if err != nil { + return ErrorResult(fmt.Sprintf("Video analysis failed: %v", err)) + } + + result := NewResult(string(chainResult.Data)) + result.Usage = chainResult.Usage + result.Provider = chainResult.Provider + result.Model = chainResult.Model return result } diff --git a/internal/tools/read_video_resolve.go b/internal/tools/read_video_resolve.go index ef905d73..f6a6b13e 100644 --- a/internal/tools/read_video_resolve.go +++ b/internal/tools/read_video_resolve.go @@ -3,7 +3,6 @@ package tools import ( "context" "encoding/base64" - "encoding/json" "fmt" "log/slog" "path/filepath" @@ -13,16 +12,6 @@ import ( "github.com/nextlevelbuilder/goclaw/internal/providers" ) -// videoProviderPriority is the order in which providers are tried for video analysis. -// OpenAI excluded — no native video upload in chat completions. -var videoProviderPriority = []string{"gemini", "openrouter"} - -// videoModelOverrides maps provider names to preferred video-capable models. -var videoModelOverrides = map[string]string{ - "gemini": "gemini-2.5-flash", - "openrouter": "google/gemini-2.5-flash", -} - // resolveVideoFile finds the video file path from context MediaRefs. func (t *ReadVideoTool) resolveVideoFile(ctx context.Context, mediaID string) (path, mime string, err error) { if t.mediaLoader == nil { @@ -62,97 +51,32 @@ func (t *ReadVideoTool) resolveVideoFile(ctx context.Context, mediaID string) (p return p, mime, nil } -// resolveVideoProviderWithConfig checks builtin settings, then hardcoded priority. -func (t *ReadVideoTool) resolveVideoProviderWithConfig(ctx context.Context) (providers.Provider, string, error) { - if p, model, ok := t.resolveFromVideoSettings(ctx); ok { - return p, model, nil - } - return t.resolveVideoProvider() -} - -// resolveFromVideoSettings checks global builtin tool settings for provider/model config. -func (t *ReadVideoTool) resolveFromVideoSettings(ctx context.Context) (providers.Provider, string, bool) { - settings := BuiltinToolSettingsFromCtx(ctx) - if settings == nil { - return nil, "", false - } - raw, ok := settings["read_video"] - if !ok || len(raw) == 0 { - return nil, "", false - } - var cfg struct { - Provider string `json:"provider"` - Model string `json:"model"` - } - if err := json.Unmarshal(raw, &cfg); err != nil || cfg.Provider == "" { - return nil, "", false - } - p, err := t.registry.Get(cfg.Provider) - if err != nil { - return nil, "", false - } - model := cfg.Model - if model == "" { - model = p.DefaultModel() - } - return p, model, true -} - -// resolveVideoProvider finds the first available video-capable provider. -func (t *ReadVideoTool) resolveVideoProvider() (providers.Provider, string, error) { - for _, name := range videoProviderPriority { - p, err := t.registry.Get(name) - if err != nil { - continue - } - model := p.DefaultModel() - if override, ok := videoModelOverrides[name]; ok { - model = override - } - return p, model, nil - } - return nil, "", fmt.Errorf("no video-capable provider available (need one of: %v)", videoProviderPriority) -} - -// resolveVideoProviderByName gets a specific provider by name and applies model override. -func (t *ReadVideoTool) resolveVideoProviderByName(name string) (providers.Provider, string, error) { - p, err := t.registry.Get(name) - if err != nil { - return nil, "", err - } - model := p.DefaultModel() - if override, ok := videoModelOverrides[name]; ok { - model = override - } - return p, model, nil -} - -// callVideoProvider sends video to a provider for analysis. +// callProvider dispatches video analysis to the appropriate provider API. // Gemini: uses File API (upload → poll → file_data in generateContent). // Others: falls back to base64 in image_url (OpenRouter routes to Gemini which handles video). -func (t *ReadVideoTool) callVideoProvider(ctx context.Context, provider providers.Provider, model, prompt string, data []byte, mime string) (*providers.ChatResponse, string, string) { - provName := provider.Name() +func (t *ReadVideoTool) callProvider(ctx context.Context, cp credentialProvider, providerName, model string, params map[string]any) ([]byte, *providers.Usage, error) { + prompt := GetParamString(params, "prompt", "Analyze this video and describe its contents.") + data, _ := params["data"].([]byte) + mime := GetParamString(params, "mime", "video/mp4") // Gemini: use File API. - if strings.HasPrefix(provName, "gemini") { - oaiProv, ok := provider.(*providers.OpenAIProvider) - if !ok { - slog.Warn("read_video: gemini provider is not OpenAIProvider", "provider", provName) - return nil, "", "" - } - apiKey := oaiProv.APIKey() - slog.Info("read_video: using gemini file API", "provider", provName, "model", model, "size", len(data), "mime", mime) - resp, err := geminiFileAPICall(ctx, apiKey, model, prompt, data, mime, 180*time.Second) + if strings.HasPrefix(providerName, "gemini") { + slog.Info("read_video: using gemini file API", "provider", providerName, "model", model, "size", len(data), "mime", mime) + resp, err := geminiFileAPICall(ctx, cp.APIKey(), model, prompt, data, mime, 180*time.Second) if err != nil { - slog.Warn("read_video: gemini file API call failed", "error", err) - return nil, "", "" + return nil, nil, fmt.Errorf("gemini file API: %w", err) } - return resp, provName, model + return []byte(resp.Content), resp.Usage, nil } // Other providers: try standard Chat API with base64 as image_url (best effort). - slog.Info("read_video: using chat API fallback", "provider", provName, "model", model, "size", len(data)) - resp, err := provider.Chat(ctx, providers.ChatRequest{ + p, err := t.registry.Get(providerName) + if err != nil { + return nil, nil, fmt.Errorf("provider %q not available: %w", providerName, err) + } + + slog.Info("read_video: using chat API fallback", "provider", providerName, "model", model, "size", len(data)) + resp, err := p.Chat(ctx, providers.ChatRequest{ Messages: []providers.Message{ { Role: "user", @@ -167,8 +91,7 @@ func (t *ReadVideoTool) callVideoProvider(ctx context.Context, provider provider }, }) if err != nil { - slog.Warn("read_video: chat API fallback failed", "provider", provName, "error", err) - return nil, "", "" + return nil, nil, fmt.Errorf("chat API: %w", err) } - return resp, provName, model + return []byte(resp.Content), resp.Usage, nil }