mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-12 22:14:37 +00:00
refactor(tools): migrate read_* tools to provider chain and add media models
Refactor read_image, read_document, read_video, read_audio to use ResolveMediaProviderChain + ExecuteWithChain for consistent fallback behavior. Add hardcoded model lists for MiniMax, DashScope, and Suno providers.
This commit is contained in:
1 parent
6345df3136
commit
01d75ac7fe
8 files changed
+240
-480
No files matched your search
@@ -60,8 +60,12 @@ func (h *ProvidersHandler) handleListProviderModels(w http.ResponseWriter, r *ht
|
||||
models, err = fetchGeminiModels(ctx, p.APIKey)
|
||||
case "bailian":
|
||||
models = bailianModels()
|
||||
case "dashscope":
|
||||
models = dashScopeModels()
|
||||
case "minimax_native":
|
||||
models = minimaxModels()
|
||||
case "suno":
|
||||
models = sunoModels()
|
||||
default:
|
||||
// All other types use OpenAI-compatible /models endpoint
|
||||
apiBase := strings.TrimRight(p.APIBase, "/")
|
||||
@@ -176,9 +180,47 @@ func bailianModels() []ModelInfo {
|
||||
// MiniMax does not expose a /v1/models endpoint.
|
||||
func minimaxModels() []ModelInfo {
|
||||
return []ModelInfo{
|
||||
// Chat / text
|
||||
{ID: "MiniMax-Text-01", Name: "MiniMax Text 01"},
|
||||
{ID: "MiniMax-M1", Name: "MiniMax M1"},
|
||||
{ID: "MiniMax-M2.5", Name: "MiniMax M2.5"},
|
||||
// Image generation
|
||||
{ID: "image-01", Name: "Image 01"},
|
||||
// Video generation
|
||||
{ID: "MiniMax-Hailuo-2.3", Name: "Hailuo Video 2.3"},
|
||||
{ID: "MiniMax-Hailuo-2", Name: "Hailuo Video 2"},
|
||||
{ID: "T2V-01-Director", Name: "T2V-01 Director"},
|
||||
// Music generation
|
||||
{ID: "music-2.5+", Name: "Music 2.5+"},
|
||||
{ID: "music-2.5", Name: "Music 2.5"},
|
||||
// TTS
|
||||
{ID: "speech-02-hd", Name: "Speech 02 HD"},
|
||||
{ID: "speech-02-turbo", Name: "Speech 02 Turbo"},
|
||||
}
|
||||
}
|
||||
|
||||
// dashScopeModels returns a hardcoded list of DashScope (Qwen) models.
|
||||
// DashScope does not expose a standard /v1/models endpoint.
|
||||
func dashScopeModels() []ModelInfo {
|
||||
return []ModelInfo{
|
||||
// Chat / text
|
||||
{ID: "qwen3-max", Name: "Qwen 3 Max"},
|
||||
{ID: "qwen3-plus", Name: "Qwen 3 Plus"},
|
||||
{ID: "qwen3-turbo", Name: "Qwen 3 Turbo"},
|
||||
// Image generation
|
||||
{ID: "wan2.6-image", Name: "Wan 2.6 Image"},
|
||||
{ID: "wan2.1-image", Name: "Wan 2.1 Image"},
|
||||
// Video generation
|
||||
{ID: "wan2.6-video", Name: "Wan 2.6 Video"},
|
||||
}
|
||||
}
|
||||
|
||||
// sunoModels returns a hardcoded list of Suno music generation models.
|
||||
func sunoModels() []ModelInfo {
|
||||
return []ModelInfo{
|
||||
{ID: "v4.5", Name: "Suno V4.5"},
|
||||
{ID: "v4", Name: "Suno V4"},
|
||||
{ID: "v3.5", Name: "Suno V3.5"},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -30,8 +30,18 @@ func MediaAudioRefsFromCtx(ctx context.Context) []providers.MediaRef {
|
||||
// audioMaxBytes is the max file size for audio analysis (50MB).
|
||||
const audioMaxBytes = 50 * 1024 * 1024
|
||||
|
||||
// audioProviderPriority is the order in which providers are tried for audio analysis.
|
||||
var audioProviderPriority = []string{"gemini", "openai", "openrouter"}
|
||||
|
||||
// audioModelDefaults maps provider names to preferred audio-capable models.
|
||||
var audioModelDefaults = map[string]string{
|
||||
"gemini": "gemini-2.5-flash",
|
||||
"openai": "gpt-4o-audio-preview",
|
||||
"openrouter": "google/gemini-2.5-flash",
|
||||
}
|
||||
|
||||
// ReadAudioTool uses an audio-capable provider to analyze audio files
|
||||
// attached to the current conversation. Follows same pattern as ReadDocumentTool.
|
||||
// attached to the current conversation.
|
||||
type ReadAudioTool struct {
|
||||
registry *providers.Registry
|
||||
mediaLoader MediaPathLoader
|
||||
@@ -73,7 +83,6 @@ func (t *ReadAudioTool) Execute(ctx context.Context, args map[string]interface{}
|
||||
}
|
||||
mediaID, _ := args["media_id"].(string)
|
||||
|
||||
// Resolve audio file path from MediaRefs in context.
|
||||
audioPath, audioMime, err := t.resolveAudioFile(ctx, mediaID)
|
||||
if err != nil {
|
||||
return ErrorResult(err.Error())
|
||||
@@ -81,7 +90,6 @@ func (t *ReadAudioTool) Execute(ctx context.Context, args map[string]interface{}
|
||||
|
||||
slog.Info("read_audio: resolved file", "path", audioPath, "mime", audioMime, "media_id", mediaID)
|
||||
|
||||
// Read audio file.
|
||||
data, err := os.ReadFile(audioPath)
|
||||
if err != nil {
|
||||
return ErrorResult(fmt.Sprintf("Failed to read audio file: %v", err))
|
||||
@@ -91,40 +99,27 @@ func (t *ReadAudioTool) Execute(ctx context.Context, args map[string]interface{}
|
||||
return ErrorResult(fmt.Sprintf("Audio too large: %d bytes (max %d)", len(data), audioMaxBytes))
|
||||
}
|
||||
|
||||
// Find an audio-capable provider.
|
||||
provider, model, err := t.resolveAudioProviderWithConfig(ctx)
|
||||
if err != nil {
|
||||
return ErrorResult(err.Error())
|
||||
}
|
||||
chain := ResolveMediaProviderChain(ctx, "read_audio", "", "",
|
||||
audioProviderPriority, audioModelDefaults, t.registry)
|
||||
|
||||
// Try primary provider, fallback to next available on error.
|
||||
resp, usedProvider, usedModel := t.callAudioProvider(ctx, provider, model, prompt, data, audioMime)
|
||||
if resp == nil {
|
||||
// Primary failed — try fallback providers from priority list.
|
||||
slog.Warn("read_audio: primary provider failed, trying fallback", "primary", provider.Name())
|
||||
for _, fbName := range audioProviderPriority {
|
||||
if fbName == provider.Name() {
|
||||
continue
|
||||
}
|
||||
fbProvider, fbModel, fbErr := t.resolveAudioProviderByName(fbName)
|
||||
if fbErr != nil {
|
||||
continue
|
||||
}
|
||||
resp, usedProvider, usedModel = t.callAudioProvider(ctx, fbProvider, fbModel, prompt, data, audioMime)
|
||||
if resp != nil {
|
||||
slog.Info("read_audio: fallback succeeded", "provider", usedProvider)
|
||||
break
|
||||
}
|
||||
for i := range chain {
|
||||
if chain[i].Params == nil {
|
||||
chain[i].Params = make(map[string]any)
|
||||
}
|
||||
}
|
||||
if resp == nil {
|
||||
return ErrorResult("Audio analysis failed: all providers returned errors")
|
||||
chain[i].Params["prompt"] = prompt
|
||||
chain[i].Params["data"] = data
|
||||
chain[i].Params["mime"] = audioMime
|
||||
}
|
||||
|
||||
result := NewResult(resp.Content)
|
||||
result.Usage = resp.Usage
|
||||
result.Provider = usedProvider
|
||||
result.Model = usedModel
|
||||
chainResult, err := ExecuteWithChain(ctx, chain, t.registry, t.callProvider)
|
||||
if err != nil {
|
||||
return ErrorResult(fmt.Sprintf("Audio analysis failed: %v", err))
|
||||
}
|
||||
|
||||
result := NewResult(string(chainResult.Data))
|
||||
result.Usage = chainResult.Usage
|
||||
result.Provider = chainResult.Provider
|
||||
result.Model = chainResult.Model
|
||||
return result
|
||||
}
|
||||
|
||||
|
||||
@@ -3,7 +3,6 @@ package tools
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"path/filepath"
|
||||
@@ -13,16 +12,6 @@ import (
|
||||
"github.com/nextlevelbuilder/goclaw/internal/providers"
|
||||
)
|
||||
|
||||
// audioProviderPriority is the order in which providers are tried for audio analysis.
|
||||
var audioProviderPriority = []string{"gemini", "openai", "openrouter"}
|
||||
|
||||
// audioModelOverrides maps provider names to preferred audio-capable models.
|
||||
var audioModelOverrides = map[string]string{
|
||||
"gemini": "gemini-2.5-flash",
|
||||
"openai": "gpt-4o-audio-preview",
|
||||
"openrouter": "google/gemini-2.5-flash",
|
||||
}
|
||||
|
||||
// resolveAudioFile finds the audio file path from context MediaRefs.
|
||||
func (t *ReadAudioTool) resolveAudioFile(ctx context.Context, mediaID string) (path, mime string, err error) {
|
||||
if t.mediaLoader == nil {
|
||||
@@ -62,114 +51,43 @@ func (t *ReadAudioTool) resolveAudioFile(ctx context.Context, mediaID string) (p
|
||||
return p, mime, nil
|
||||
}
|
||||
|
||||
// resolveAudioProviderWithConfig checks builtin settings, then hardcoded priority.
|
||||
func (t *ReadAudioTool) resolveAudioProviderWithConfig(ctx context.Context) (providers.Provider, string, error) {
|
||||
if p, model, ok := t.resolveFromAudioSettings(ctx); ok {
|
||||
return p, model, nil
|
||||
}
|
||||
return t.resolveAudioProvider()
|
||||
}
|
||||
|
||||
// resolveFromAudioSettings checks global builtin tool settings for provider/model config.
|
||||
func (t *ReadAudioTool) resolveFromAudioSettings(ctx context.Context) (providers.Provider, string, bool) {
|
||||
settings := BuiltinToolSettingsFromCtx(ctx)
|
||||
if settings == nil {
|
||||
return nil, "", false
|
||||
}
|
||||
raw, ok := settings["read_audio"]
|
||||
if !ok || len(raw) == 0 {
|
||||
return nil, "", false
|
||||
}
|
||||
var cfg struct {
|
||||
Provider string `json:"provider"`
|
||||
Model string `json:"model"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &cfg); err != nil || cfg.Provider == "" {
|
||||
return nil, "", false
|
||||
}
|
||||
p, err := t.registry.Get(cfg.Provider)
|
||||
if err != nil {
|
||||
return nil, "", false
|
||||
}
|
||||
model := cfg.Model
|
||||
if model == "" {
|
||||
model = p.DefaultModel()
|
||||
}
|
||||
return p, model, true
|
||||
}
|
||||
|
||||
// resolveAudioProvider finds the first available audio-capable provider.
|
||||
func (t *ReadAudioTool) resolveAudioProvider() (providers.Provider, string, error) {
|
||||
for _, name := range audioProviderPriority {
|
||||
p, err := t.registry.Get(name)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
model := p.DefaultModel()
|
||||
if override, ok := audioModelOverrides[name]; ok {
|
||||
model = override
|
||||
}
|
||||
return p, model, nil
|
||||
}
|
||||
return nil, "", fmt.Errorf("no audio-capable provider available (need one of: %v)", audioProviderPriority)
|
||||
}
|
||||
|
||||
// resolveAudioProviderByName gets a specific provider by name and applies model override.
|
||||
func (t *ReadAudioTool) resolveAudioProviderByName(name string) (providers.Provider, string, error) {
|
||||
p, err := t.registry.Get(name)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
model := p.DefaultModel()
|
||||
if override, ok := audioModelOverrides[name]; ok {
|
||||
model = override
|
||||
}
|
||||
return p, model, nil
|
||||
}
|
||||
|
||||
// callAudioProvider sends audio to a provider for analysis.
|
||||
// callProvider dispatches audio analysis to the appropriate provider API.
|
||||
// Gemini: uses File API (upload → poll → file_data in generateContent).
|
||||
// OpenAI: uses input_audio content part in chat completions.
|
||||
// Others: falls back to base64 in image_url (may not work for all).
|
||||
func (t *ReadAudioTool) callAudioProvider(ctx context.Context, provider providers.Provider, model, prompt string, data []byte, mime string) (*providers.ChatResponse, string, string) {
|
||||
provName := provider.Name()
|
||||
// Others: falls back to base64 in image_url (best effort).
|
||||
func (t *ReadAudioTool) callProvider(ctx context.Context, cp credentialProvider, providerName, model string, params map[string]any) ([]byte, *providers.Usage, error) {
|
||||
prompt := GetParamString(params, "prompt", "Analyze this audio and describe its contents.")
|
||||
data, _ := params["data"].([]byte)
|
||||
mime := GetParamString(params, "mime", "audio/mpeg")
|
||||
|
||||
// Gemini: use File API (inlineData doesn't work for audio).
|
||||
if strings.HasPrefix(provName, "gemini") {
|
||||
oaiProv, ok := provider.(*providers.OpenAIProvider)
|
||||
if !ok {
|
||||
slog.Warn("read_audio: gemini provider is not OpenAIProvider", "provider", provName)
|
||||
return nil, "", ""
|
||||
}
|
||||
apiKey := oaiProv.APIKey()
|
||||
slog.Info("read_audio: using gemini file API", "provider", provName, "model", model, "size", len(data), "mime", mime)
|
||||
resp, err := geminiFileAPICall(ctx, apiKey, model, prompt, data, mime, 120*time.Second)
|
||||
if strings.HasPrefix(providerName, "gemini") {
|
||||
slog.Info("read_audio: using gemini file API", "provider", providerName, "model", model, "size", len(data), "mime", mime)
|
||||
resp, err := geminiFileAPICall(ctx, cp.APIKey(), model, prompt, data, mime, 120*time.Second)
|
||||
if err != nil {
|
||||
slog.Warn("read_audio: gemini file API call failed", "error", err)
|
||||
return nil, "", ""
|
||||
return nil, nil, fmt.Errorf("gemini file API: %w", err)
|
||||
}
|
||||
return resp, provName, model
|
||||
return []byte(resp.Content), resp.Usage, nil
|
||||
}
|
||||
|
||||
// OpenAI: use input_audio content part (supports wav, mp3).
|
||||
if strings.HasPrefix(provName, "openai") {
|
||||
oaiProv, ok := provider.(*providers.OpenAIProvider)
|
||||
if !ok {
|
||||
slog.Warn("read_audio: openai provider is not OpenAIProvider", "provider", provName)
|
||||
return nil, "", ""
|
||||
}
|
||||
slog.Info("read_audio: using openai input_audio API", "provider", provName, "model", model, "size", len(data), "mime", mime)
|
||||
resp, err := openaiAudioCall(ctx, oaiProv.APIKey(), oaiProv.APIBase(), model, prompt, data, mime)
|
||||
if strings.HasPrefix(providerName, "openai") {
|
||||
slog.Info("read_audio: using openai input_audio API", "provider", providerName, "model", model, "size", len(data), "mime", mime)
|
||||
resp, err := openaiAudioCall(ctx, cp.APIKey(), cp.APIBase(), model, prompt, data, mime)
|
||||
if err != nil {
|
||||
slog.Warn("read_audio: openai audio call failed", "error", err)
|
||||
return nil, "", ""
|
||||
return nil, nil, fmt.Errorf("openai audio call: %w", err)
|
||||
}
|
||||
return resp, provName, model
|
||||
return []byte(resp.Content), resp.Usage, nil
|
||||
}
|
||||
|
||||
// Other providers: try standard Chat API with base64 audio as image_url (best effort).
|
||||
slog.Info("read_audio: using chat API fallback", "provider", provName, "model", model, "size", len(data))
|
||||
resp, err := provider.Chat(ctx, providers.ChatRequest{
|
||||
p, err := t.registry.Get(providerName)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("provider %q not available: %w", providerName, err)
|
||||
}
|
||||
|
||||
slog.Info("read_audio: using chat API fallback", "provider", providerName, "model", model, "size", len(data))
|
||||
resp, err := p.Chat(ctx, providers.ChatRequest{
|
||||
Messages: []providers.Message{
|
||||
{
|
||||
Role: "user",
|
||||
@@ -184,10 +102,9 @@ func (t *ReadAudioTool) callAudioProvider(ctx context.Context, provider provider
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
slog.Warn("read_audio: chat API fallback failed", "provider", provName, "error", err)
|
||||
return nil, "", ""
|
||||
return nil, nil, fmt.Errorf("chat API: %w", err)
|
||||
}
|
||||
return resp, provName, model
|
||||
return []byte(resp.Content), resp.Usage, nil
|
||||
}
|
||||
|
||||
// openaiAudioCall sends audio to OpenAI using the input_audio content part.
|
||||
|
||||
@@ -48,6 +48,17 @@ func MediaDocRefsFromCtx(ctx context.Context) []providers.MediaRef {
|
||||
// documentMaxBytes is the max file size for document analysis (20MB).
|
||||
const documentMaxBytes = 20 * 1024 * 1024
|
||||
|
||||
// documentProviderPriority is the order in which providers are tried for document analysis.
|
||||
// Gemini has best native PDF support (50MB, 258 tokens/page).
|
||||
var documentProviderPriority = []string{"gemini", "anthropic", "openrouter", "dashscope"}
|
||||
|
||||
// documentModelDefaults maps provider names to preferred document-capable models.
|
||||
var documentModelDefaults = map[string]string{
|
||||
"gemini": "gemini-2.5-flash",
|
||||
"openrouter": "google/gemini-2.5-flash",
|
||||
"dashscope": "qwen-vl-max",
|
||||
}
|
||||
|
||||
// ReadDocumentTool uses a document-capable provider to analyze files
|
||||
// attached to the current conversation. Follows same pattern as ReadImageTool.
|
||||
type ReadDocumentTool struct {
|
||||
@@ -119,40 +130,27 @@ func (t *ReadDocumentTool) Execute(ctx context.Context, args map[string]interfac
|
||||
return NewResult(content)
|
||||
}
|
||||
|
||||
// Find a document-capable provider.
|
||||
provider, model, err := t.resolveDocumentProviderWithConfig(ctx)
|
||||
if err != nil {
|
||||
return ErrorResult(err.Error())
|
||||
}
|
||||
chain := ResolveMediaProviderChain(ctx, "read_document", "", "",
|
||||
documentProviderPriority, documentModelDefaults, t.registry)
|
||||
|
||||
// Try primary provider, fallback to next available on error.
|
||||
resp, usedProvider, usedModel := t.callDocumentProvider(ctx, provider, model, prompt, data, docMime)
|
||||
if resp == nil {
|
||||
// Primary failed — try fallback providers from priority list.
|
||||
slog.Warn("read_document: primary provider failed, trying fallback", "primary", provider.Name())
|
||||
for _, fbName := range documentProviderPriority {
|
||||
if fbName == provider.Name() {
|
||||
continue // skip the one that already failed
|
||||
}
|
||||
fbProvider, fbModel, fbErr := t.resolveDocumentProviderByName(fbName)
|
||||
if fbErr != nil {
|
||||
continue
|
||||
}
|
||||
resp, usedProvider, usedModel = t.callDocumentProvider(ctx, fbProvider, fbModel, prompt, data, docMime)
|
||||
if resp != nil {
|
||||
slog.Info("read_document: fallback succeeded", "provider", usedProvider)
|
||||
break
|
||||
}
|
||||
// Inject prompt, data, and mime into each chain entry's params
|
||||
for i := range chain {
|
||||
if chain[i].Params == nil {
|
||||
chain[i].Params = make(map[string]any)
|
||||
}
|
||||
}
|
||||
if resp == nil {
|
||||
return ErrorResult("Document analysis failed: all providers returned errors")
|
||||
chain[i].Params["prompt"] = prompt
|
||||
chain[i].Params["data"] = data
|
||||
chain[i].Params["mime"] = docMime
|
||||
}
|
||||
|
||||
result := NewResult(resp.Content)
|
||||
result.Usage = resp.Usage
|
||||
result.Provider = usedProvider
|
||||
result.Model = usedModel
|
||||
chainResult, err := ExecuteWithChain(ctx, chain, t.registry, t.callProvider)
|
||||
if err != nil {
|
||||
return ErrorResult(fmt.Sprintf("Document analysis failed: %v", err))
|
||||
}
|
||||
|
||||
result := NewResult(string(chainResult.Data))
|
||||
result.Usage = chainResult.Usage
|
||||
result.Provider = chainResult.Provider
|
||||
result.Model = chainResult.Model
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package tools
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"path/filepath"
|
||||
@@ -12,17 +11,6 @@ import (
|
||||
"github.com/nextlevelbuilder/goclaw/internal/providers"
|
||||
)
|
||||
|
||||
// documentProviderPriority is the order in which providers are tried for document analysis.
|
||||
// Gemini has best native PDF support (50MB, 258 tokens/page).
|
||||
var documentProviderPriority = []string{"gemini", "anthropic", "openrouter", "dashscope"}
|
||||
|
||||
// documentModelOverrides maps provider names to preferred document-capable models.
|
||||
var documentModelOverrides = map[string]string{
|
||||
"gemini": "gemini-2.5-flash",
|
||||
"openrouter": "google/gemini-2.5-flash",
|
||||
"dashscope": "qwen-vl-max",
|
||||
}
|
||||
|
||||
// resolveDocumentFile finds the document file path from context MediaRefs.
|
||||
func (t *ReadDocumentTool) resolveDocumentFile(ctx context.Context, mediaID string) (path, mime string, err error) {
|
||||
if t.mediaLoader == nil {
|
||||
@@ -65,88 +53,34 @@ func (t *ReadDocumentTool) resolveDocumentFile(ctx context.Context, mediaID stri
|
||||
return p, mime, nil
|
||||
}
|
||||
|
||||
// resolveDocumentProviderWithConfig checks per-agent config, global settings, then hardcoded priority.
|
||||
func (t *ReadDocumentTool) resolveDocumentProviderWithConfig(ctx context.Context) (providers.Provider, string, error) {
|
||||
// 1. Global builtin_tools.settings
|
||||
if p, model, ok := t.resolveFromBuiltinSettings(ctx); ok {
|
||||
return p, model, nil
|
||||
}
|
||||
// 2. Hardcoded defaults
|
||||
return t.resolveDocumentProvider()
|
||||
}
|
||||
|
||||
// resolveFromBuiltinSettings checks global builtin tool settings for provider/model config.
|
||||
func (t *ReadDocumentTool) resolveFromBuiltinSettings(ctx context.Context) (providers.Provider, string, bool) {
|
||||
settings := BuiltinToolSettingsFromCtx(ctx)
|
||||
if settings == nil {
|
||||
return nil, "", false
|
||||
}
|
||||
raw, ok := settings["read_document"]
|
||||
if !ok || len(raw) == 0 {
|
||||
return nil, "", false
|
||||
}
|
||||
var cfg struct {
|
||||
Provider string `json:"provider"`
|
||||
Model string `json:"model"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &cfg); err != nil || cfg.Provider == "" {
|
||||
return nil, "", false
|
||||
}
|
||||
p, err := t.registry.Get(cfg.Provider)
|
||||
if err != nil {
|
||||
return nil, "", false
|
||||
}
|
||||
model := cfg.Model
|
||||
if model == "" {
|
||||
model = p.DefaultModel()
|
||||
}
|
||||
return p, model, true
|
||||
}
|
||||
|
||||
// resolveDocumentProvider finds the first available document-capable provider.
|
||||
func (t *ReadDocumentTool) resolveDocumentProvider() (providers.Provider, string, error) {
|
||||
for _, name := range documentProviderPriority {
|
||||
p, err := t.registry.Get(name)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
model := p.DefaultModel()
|
||||
if override, ok := documentModelOverrides[name]; ok {
|
||||
model = override
|
||||
}
|
||||
return p, model, nil
|
||||
}
|
||||
return nil, "", fmt.Errorf("no document-capable provider available (need one of: %v)", documentProviderPriority)
|
||||
}
|
||||
|
||||
// callDocumentProvider sends a document to a provider for analysis.
|
||||
// For Gemini providers, uses native generateContent API (supports PDF natively).
|
||||
// For others, falls back to OpenAI-compat chat with base64 document.
|
||||
func (t *ReadDocumentTool) callDocumentProvider(ctx context.Context, provider providers.Provider, model, prompt string, data []byte, mime string) (*providers.ChatResponse, string, string) {
|
||||
provName := provider.Name()
|
||||
// callProvider dispatches document analysis to the appropriate provider API.
|
||||
// For Gemini: uses native generateContent API (supports PDF natively).
|
||||
// For others: uses standard Chat API with base64 document.
|
||||
func (t *ReadDocumentTool) callProvider(ctx context.Context, cp credentialProvider, providerName, model string, params map[string]any) ([]byte, *providers.Usage, error) {
|
||||
prompt := GetParamString(params, "prompt", "Analyze this document and describe its contents.")
|
||||
data, _ := params["data"].([]byte)
|
||||
mime := GetParamString(params, "mime", "application/octet-stream")
|
||||
|
||||
// Gemini: use native API (OpenAI-compat endpoint doesn't support non-image MIME types).
|
||||
if strings.HasPrefix(provName, "gemini") {
|
||||
oaiProv, ok := provider.(*providers.OpenAIProvider)
|
||||
if !ok {
|
||||
slog.Warn("read_document: gemini provider is not OpenAIProvider", "provider", provName)
|
||||
return nil, "", ""
|
||||
}
|
||||
apiKey := oaiProv.APIKey()
|
||||
if strings.HasPrefix(providerName, "gemini") {
|
||||
slog.Info("read_document: using gemini native API",
|
||||
"provider", provName, "model", model,
|
||||
"provider", providerName, "model", model,
|
||||
"doc_size", len(data), "mime", mime)
|
||||
resp, err := geminiNativeDocumentCall(ctx, apiKey, model, prompt, data, mime)
|
||||
resp, err := geminiNativeDocumentCall(ctx, cp.APIKey(), model, prompt, data, mime)
|
||||
if err != nil {
|
||||
slog.Warn("read_document: gemini native call failed", "error", err)
|
||||
return nil, "", ""
|
||||
return nil, nil, fmt.Errorf("gemini native call: %w", err)
|
||||
}
|
||||
return resp, provName, model
|
||||
return []byte(resp.Content), resp.Usage, nil
|
||||
}
|
||||
|
||||
// Other providers: use standard Chat API with document as base64 image_url.
|
||||
slog.Info("read_document: using chat API", "provider", provName, "model", model, "doc_size", len(data))
|
||||
resp, err := provider.Chat(ctx, providers.ChatRequest{
|
||||
p, err := t.registry.Get(providerName)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("provider %q not available: %w", providerName, err)
|
||||
}
|
||||
|
||||
slog.Info("read_document: using chat API", "provider", providerName, "model", model, "doc_size", len(data))
|
||||
resp, err := p.Chat(ctx, providers.ChatRequest{
|
||||
Messages: []providers.Message{
|
||||
{
|
||||
Role: "user",
|
||||
@@ -161,23 +95,9 @@ func (t *ReadDocumentTool) callDocumentProvider(ctx context.Context, provider pr
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
slog.Warn("read_document: chat call failed", "provider", provName, "error", err)
|
||||
return nil, "", ""
|
||||
return nil, nil, fmt.Errorf("chat call: %w", err)
|
||||
}
|
||||
return resp, provName, model
|
||||
}
|
||||
|
||||
// resolveDocumentProviderByName gets a specific provider by name and applies model override.
|
||||
func (t *ReadDocumentTool) resolveDocumentProviderByName(name string) (providers.Provider, string, error) {
|
||||
p, err := t.registry.Get(name)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
model := p.DefaultModel()
|
||||
if override, ok := documentModelOverrides[name]; ok {
|
||||
model = override
|
||||
}
|
||||
return p, model, nil
|
||||
return []byte(resp.Content), resp.Usage, nil
|
||||
}
|
||||
|
||||
// mimeFromDocExt returns MIME type for document file extensions.
|
||||
|
||||
@@ -2,7 +2,6 @@ package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
|
||||
@@ -29,10 +28,11 @@ func MediaImagesFromCtx(ctx context.Context) []providers.ImageContent {
|
||||
// visionProviderPriority is the order in which providers are tried for vision.
|
||||
var visionProviderPriority = []string{"openrouter", "gemini", "anthropic", "dashscope"}
|
||||
|
||||
// visionModelOverrides maps provider names to preferred vision models.
|
||||
// Providers not listed here use their default model.
|
||||
var visionModelOverrides = map[string]string{
|
||||
// visionModelDefaults maps provider names to preferred vision models.
|
||||
var visionModelDefaults = map[string]string{
|
||||
"openrouter": "google/gemini-2.5-flash-image",
|
||||
"gemini": "gemini-2.5-flash",
|
||||
"anthropic": "",
|
||||
"dashscope": "qwen3-vl",
|
||||
}
|
||||
|
||||
@@ -75,15 +75,51 @@ func (t *ReadImageTool) Execute(ctx context.Context, args map[string]interface{}
|
||||
return ErrorResult("No images available in this conversation. The user may not have sent an image.")
|
||||
}
|
||||
|
||||
// Find a vision-capable provider (per-agent config > hardcoded priority)
|
||||
provider, model, err := t.resolveVisionProviderWithConfig(ctx)
|
||||
if err != nil {
|
||||
return ErrorResult(err.Error())
|
||||
// Extract per-agent config for backward compat
|
||||
var perAgentProvider, perAgentModel string
|
||||
if cfg := VisionConfigFromCtx(ctx); cfg != nil {
|
||||
perAgentProvider = cfg.Provider
|
||||
perAgentModel = cfg.Model
|
||||
}
|
||||
|
||||
slog.Info("read_image: calling vision provider", "provider", provider.Name(), "model", model, "images", len(images))
|
||||
chain := ResolveMediaProviderChain(ctx, "read_image", perAgentProvider, perAgentModel,
|
||||
visionProviderPriority, visionModelDefaults, t.registry)
|
||||
|
||||
resp, err := provider.Chat(ctx, providers.ChatRequest{
|
||||
// Inject prompt and images into each chain entry's params
|
||||
for i := range chain {
|
||||
if chain[i].Params == nil {
|
||||
chain[i].Params = make(map[string]any)
|
||||
}
|
||||
chain[i].Params["prompt"] = prompt
|
||||
chain[i].Params["images"] = images
|
||||
}
|
||||
|
||||
chainResult, err := ExecuteWithChain(ctx, chain, t.registry, t.callProvider)
|
||||
if err != nil {
|
||||
return ErrorResult(fmt.Sprintf("image analysis failed: %v", err))
|
||||
}
|
||||
|
||||
result := NewResult(string(chainResult.Data))
|
||||
result.Usage = chainResult.Usage
|
||||
result.Provider = chainResult.Provider
|
||||
result.Model = chainResult.Model
|
||||
return result
|
||||
}
|
||||
|
||||
// callProvider dispatches the vision call using provider.Chat().
|
||||
func (t *ReadImageTool) callProvider(ctx context.Context, cp credentialProvider, providerName, model string, params map[string]any) ([]byte, *providers.Usage, error) {
|
||||
prompt := GetParamString(params, "prompt", "Describe this image in detail.")
|
||||
images, _ := params["images"].([]providers.ImageContent)
|
||||
|
||||
// Get the full provider for Chat() access
|
||||
p, err := t.registry.Get(providerName)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("provider %q not available: %w", providerName, err)
|
||||
}
|
||||
|
||||
slog.Info("read_image: calling vision provider", "provider", providerName, "model", model, "images", len(images))
|
||||
|
||||
resp, err := p.Chat(ctx, providers.ChatRequest{
|
||||
Messages: []providers.Message{
|
||||
{
|
||||
Role: "user",
|
||||
@@ -98,79 +134,8 @@ func (t *ReadImageTool) Execute(ctx context.Context, args map[string]interface{}
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return ErrorResult(fmt.Sprintf("Vision provider error: %v", err))
|
||||
return nil, nil, fmt.Errorf("vision provider error: %w", err)
|
||||
}
|
||||
|
||||
result := NewResult(resp.Content)
|
||||
result.Usage = resp.Usage
|
||||
result.Provider = provider.Name()
|
||||
result.Model = model
|
||||
return result
|
||||
}
|
||||
|
||||
// resolveVisionProviderWithConfig checks per-agent VisionConfig first,
|
||||
// then global builtin_tools.settings, then falls back to hardcoded priority.
|
||||
func (t *ReadImageTool) resolveVisionProviderWithConfig(ctx context.Context) (providers.Provider, string, error) {
|
||||
// 1. Per-agent override (highest priority)
|
||||
if cfg := VisionConfigFromCtx(ctx); cfg != nil && cfg.Provider != "" {
|
||||
p, err := t.registry.Get(cfg.Provider)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("configured vision provider %q not available: %w", cfg.Provider, err)
|
||||
}
|
||||
model := cfg.Model
|
||||
if model == "" {
|
||||
model = p.DefaultModel()
|
||||
}
|
||||
return p, model, nil
|
||||
}
|
||||
// 2. Global builtin_tools.settings (DB defaults)
|
||||
if p, model, ok := t.resolveFromBuiltinSettings(ctx); ok {
|
||||
return p, model, nil
|
||||
}
|
||||
// 3. Hardcoded defaults
|
||||
return t.resolveVisionProvider()
|
||||
}
|
||||
|
||||
// resolveFromBuiltinSettings checks global builtin tool settings for provider/model config.
|
||||
func (t *ReadImageTool) resolveFromBuiltinSettings(ctx context.Context) (providers.Provider, string, bool) {
|
||||
settings := BuiltinToolSettingsFromCtx(ctx)
|
||||
if settings == nil {
|
||||
return nil, "", false
|
||||
}
|
||||
raw, ok := settings["read_image"]
|
||||
if !ok || len(raw) == 0 {
|
||||
return nil, "", false
|
||||
}
|
||||
var cfg struct {
|
||||
Provider string `json:"provider"`
|
||||
Model string `json:"model"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &cfg); err != nil || cfg.Provider == "" {
|
||||
return nil, "", false
|
||||
}
|
||||
p, err := t.registry.Get(cfg.Provider)
|
||||
if err != nil {
|
||||
return nil, "", false
|
||||
}
|
||||
model := cfg.Model
|
||||
if model == "" {
|
||||
model = p.DefaultModel()
|
||||
}
|
||||
return p, model, true
|
||||
}
|
||||
|
||||
// resolveVisionProvider finds the first available vision-capable provider.
|
||||
func (t *ReadImageTool) resolveVisionProvider() (providers.Provider, string, error) {
|
||||
for _, name := range visionProviderPriority {
|
||||
p, err := t.registry.Get(name)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
model := p.DefaultModel()
|
||||
if override, ok := visionModelOverrides[name]; ok {
|
||||
model = override
|
||||
}
|
||||
return p, model, nil
|
||||
}
|
||||
return nil, "", fmt.Errorf("no vision-capable provider available (need one of: %v)", visionProviderPriority)
|
||||
return []byte(resp.Content), resp.Usage, nil
|
||||
}
|
||||
@@ -30,8 +30,18 @@ func MediaVideoRefsFromCtx(ctx context.Context) []providers.MediaRef {
|
||||
// videoMaxBytes is the max file size for video analysis (100MB).
|
||||
const videoMaxBytes = 100 * 1024 * 1024
|
||||
|
||||
// videoProviderPriority is the order in which providers are tried for video analysis.
|
||||
// OpenAI excluded — no native video upload in chat completions.
|
||||
var videoProviderPriority = []string{"gemini", "openrouter"}
|
||||
|
||||
// videoModelDefaults maps provider names to preferred video-capable models.
|
||||
var videoModelDefaults = map[string]string{
|
||||
"gemini": "gemini-2.5-flash",
|
||||
"openrouter": "google/gemini-2.5-flash",
|
||||
}
|
||||
|
||||
// ReadVideoTool uses a video-capable provider to analyze video files
|
||||
// attached to the current conversation. Follows same pattern as ReadAudioTool.
|
||||
// attached to the current conversation.
|
||||
type ReadVideoTool struct {
|
||||
registry *providers.Registry
|
||||
mediaLoader MediaPathLoader
|
||||
@@ -89,37 +99,27 @@ func (t *ReadVideoTool) Execute(ctx context.Context, args map[string]interface{}
|
||||
return ErrorResult(fmt.Sprintf("Video too large: %d bytes (max %d)", len(data), videoMaxBytes))
|
||||
}
|
||||
|
||||
provider, model, err := t.resolveVideoProviderWithConfig(ctx)
|
||||
if err != nil {
|
||||
return ErrorResult(err.Error())
|
||||
}
|
||||
chain := ResolveMediaProviderChain(ctx, "read_video", "", "",
|
||||
videoProviderPriority, videoModelDefaults, t.registry)
|
||||
|
||||
resp, usedProvider, usedModel := t.callVideoProvider(ctx, provider, model, prompt, data, videoMime)
|
||||
if resp == nil {
|
||||
slog.Warn("read_video: primary provider failed, trying fallback", "primary", provider.Name())
|
||||
for _, fbName := range videoProviderPriority {
|
||||
if fbName == provider.Name() {
|
||||
continue
|
||||
}
|
||||
fbProvider, fbModel, fbErr := t.resolveVideoProviderByName(fbName)
|
||||
if fbErr != nil {
|
||||
continue
|
||||
}
|
||||
resp, usedProvider, usedModel = t.callVideoProvider(ctx, fbProvider, fbModel, prompt, data, videoMime)
|
||||
if resp != nil {
|
||||
slog.Info("read_video: fallback succeeded", "provider", usedProvider)
|
||||
break
|
||||
}
|
||||
for i := range chain {
|
||||
if chain[i].Params == nil {
|
||||
chain[i].Params = make(map[string]any)
|
||||
}
|
||||
}
|
||||
if resp == nil {
|
||||
return ErrorResult("Video analysis failed: all providers returned errors")
|
||||
chain[i].Params["prompt"] = prompt
|
||||
chain[i].Params["data"] = data
|
||||
chain[i].Params["mime"] = videoMime
|
||||
}
|
||||
|
||||
result := NewResult(resp.Content)
|
||||
result.Usage = resp.Usage
|
||||
result.Provider = usedProvider
|
||||
result.Model = usedModel
|
||||
chainResult, err := ExecuteWithChain(ctx, chain, t.registry, t.callProvider)
|
||||
if err != nil {
|
||||
return ErrorResult(fmt.Sprintf("Video analysis failed: %v", err))
|
||||
}
|
||||
|
||||
result := NewResult(string(chainResult.Data))
|
||||
result.Usage = chainResult.Usage
|
||||
result.Provider = chainResult.Provider
|
||||
result.Model = chainResult.Model
|
||||
return result
|
||||
}
|
||||
|
||||
|
||||
@@ -3,7 +3,6 @@ package tools
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"path/filepath"
|
||||
@@ -13,16 +12,6 @@ import (
|
||||
"github.com/nextlevelbuilder/goclaw/internal/providers"
|
||||
)
|
||||
|
||||
// videoProviderPriority is the order in which providers are tried for video analysis.
|
||||
// OpenAI excluded — no native video upload in chat completions.
|
||||
var videoProviderPriority = []string{"gemini", "openrouter"}
|
||||
|
||||
// videoModelOverrides maps provider names to preferred video-capable models.
|
||||
var videoModelOverrides = map[string]string{
|
||||
"gemini": "gemini-2.5-flash",
|
||||
"openrouter": "google/gemini-2.5-flash",
|
||||
}
|
||||
|
||||
// resolveVideoFile finds the video file path from context MediaRefs.
|
||||
func (t *ReadVideoTool) resolveVideoFile(ctx context.Context, mediaID string) (path, mime string, err error) {
|
||||
if t.mediaLoader == nil {
|
||||
@@ -62,97 +51,32 @@ func (t *ReadVideoTool) resolveVideoFile(ctx context.Context, mediaID string) (p
|
||||
return p, mime, nil
|
||||
}
|
||||
|
||||
// resolveVideoProviderWithConfig checks builtin settings, then hardcoded priority.
|
||||
func (t *ReadVideoTool) resolveVideoProviderWithConfig(ctx context.Context) (providers.Provider, string, error) {
|
||||
if p, model, ok := t.resolveFromVideoSettings(ctx); ok {
|
||||
return p, model, nil
|
||||
}
|
||||
return t.resolveVideoProvider()
|
||||
}
|
||||
|
||||
// resolveFromVideoSettings checks global builtin tool settings for provider/model config.
|
||||
func (t *ReadVideoTool) resolveFromVideoSettings(ctx context.Context) (providers.Provider, string, bool) {
|
||||
settings := BuiltinToolSettingsFromCtx(ctx)
|
||||
if settings == nil {
|
||||
return nil, "", false
|
||||
}
|
||||
raw, ok := settings["read_video"]
|
||||
if !ok || len(raw) == 0 {
|
||||
return nil, "", false
|
||||
}
|
||||
var cfg struct {
|
||||
Provider string `json:"provider"`
|
||||
Model string `json:"model"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &cfg); err != nil || cfg.Provider == "" {
|
||||
return nil, "", false
|
||||
}
|
||||
p, err := t.registry.Get(cfg.Provider)
|
||||
if err != nil {
|
||||
return nil, "", false
|
||||
}
|
||||
model := cfg.Model
|
||||
if model == "" {
|
||||
model = p.DefaultModel()
|
||||
}
|
||||
return p, model, true
|
||||
}
|
||||
|
||||
// resolveVideoProvider finds the first available video-capable provider.
|
||||
func (t *ReadVideoTool) resolveVideoProvider() (providers.Provider, string, error) {
|
||||
for _, name := range videoProviderPriority {
|
||||
p, err := t.registry.Get(name)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
model := p.DefaultModel()
|
||||
if override, ok := videoModelOverrides[name]; ok {
|
||||
model = override
|
||||
}
|
||||
return p, model, nil
|
||||
}
|
||||
return nil, "", fmt.Errorf("no video-capable provider available (need one of: %v)", videoProviderPriority)
|
||||
}
|
||||
|
||||
// resolveVideoProviderByName gets a specific provider by name and applies model override.
|
||||
func (t *ReadVideoTool) resolveVideoProviderByName(name string) (providers.Provider, string, error) {
|
||||
p, err := t.registry.Get(name)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
model := p.DefaultModel()
|
||||
if override, ok := videoModelOverrides[name]; ok {
|
||||
model = override
|
||||
}
|
||||
return p, model, nil
|
||||
}
|
||||
|
||||
// callVideoProvider sends video to a provider for analysis.
|
||||
// callProvider dispatches video analysis to the appropriate provider API.
|
||||
// Gemini: uses File API (upload → poll → file_data in generateContent).
|
||||
// Others: falls back to base64 in image_url (OpenRouter routes to Gemini which handles video).
|
||||
func (t *ReadVideoTool) callVideoProvider(ctx context.Context, provider providers.Provider, model, prompt string, data []byte, mime string) (*providers.ChatResponse, string, string) {
|
||||
provName := provider.Name()
|
||||
func (t *ReadVideoTool) callProvider(ctx context.Context, cp credentialProvider, providerName, model string, params map[string]any) ([]byte, *providers.Usage, error) {
|
||||
prompt := GetParamString(params, "prompt", "Analyze this video and describe its contents.")
|
||||
data, _ := params["data"].([]byte)
|
||||
mime := GetParamString(params, "mime", "video/mp4")
|
||||
|
||||
// Gemini: use File API.
|
||||
if strings.HasPrefix(provName, "gemini") {
|
||||
oaiProv, ok := provider.(*providers.OpenAIProvider)
|
||||
if !ok {
|
||||
slog.Warn("read_video: gemini provider is not OpenAIProvider", "provider", provName)
|
||||
return nil, "", ""
|
||||
}
|
||||
apiKey := oaiProv.APIKey()
|
||||
slog.Info("read_video: using gemini file API", "provider", provName, "model", model, "size", len(data), "mime", mime)
|
||||
resp, err := geminiFileAPICall(ctx, apiKey, model, prompt, data, mime, 180*time.Second)
|
||||
if strings.HasPrefix(providerName, "gemini") {
|
||||
slog.Info("read_video: using gemini file API", "provider", providerName, "model", model, "size", len(data), "mime", mime)
|
||||
resp, err := geminiFileAPICall(ctx, cp.APIKey(), model, prompt, data, mime, 180*time.Second)
|
||||
if err != nil {
|
||||
slog.Warn("read_video: gemini file API call failed", "error", err)
|
||||
return nil, "", ""
|
||||
return nil, nil, fmt.Errorf("gemini file API: %w", err)
|
||||
}
|
||||
return resp, provName, model
|
||||
return []byte(resp.Content), resp.Usage, nil
|
||||
}
|
||||
|
||||
// Other providers: try standard Chat API with base64 as image_url (best effort).
|
||||
slog.Info("read_video: using chat API fallback", "provider", provName, "model", model, "size", len(data))
|
||||
resp, err := provider.Chat(ctx, providers.ChatRequest{
|
||||
p, err := t.registry.Get(providerName)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("provider %q not available: %w", providerName, err)
|
||||
}
|
||||
|
||||
slog.Info("read_video: using chat API fallback", "provider", providerName, "model", model, "size", len(data))
|
||||
resp, err := p.Chat(ctx, providers.ChatRequest{
|
||||
Messages: []providers.Message{
|
||||
{
|
||||
Role: "user",
|
||||
@@ -167,8 +91,7 @@ func (t *ReadVideoTool) callVideoProvider(ctx context.Context, provider provider
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
slog.Warn("read_video: chat API fallback failed", "provider", provName, "error", err)
|
||||
return nil, "", ""
|
||||
return nil, nil, fmt.Errorf("chat API: %w", err)
|
||||
}
|
||||
return resp, provName, model
|
||||
return []byte(resp.Content), resp.Usage, nil
|
||||
}
|
||||
Reference in new issue
Block a user