package tools import ( "context" "encoding/base64" "fmt" "log/slog" "path/filepath" "strings" "time" "github.com/nextlevelbuilder/goclaw/internal/mediabudget" "github.com/nextlevelbuilder/goclaw/internal/providers" ) // resolveAudioFile finds the audio file path from context MediaRefs. func (t *ReadAudioTool) resolveAudioFile(ctx context.Context, mediaID string) (path, mime string, err error) { refs := MediaAudioRefsFromCtx(ctx) if len(refs) == 0 { return "", "", fmt.Errorf("no audio files available in this conversation. The user may not have sent an audio file.") } var ref *providers.MediaRef if mediaID != "" { for i := range refs { if refs[i].ID == mediaID { ref = &refs[i] break } } if ref == nil { return "", "", fmt.Errorf("audio media_id %q not found in this conversation", mediaID) } } else { ref = &refs[len(refs)-1] } // Prefer persisted workspace path; fall back to legacy .media/ lookup. p := ref.Path loadedLegacy := false if p == "" { var err error if t.mediaLoader == nil { return "", "", fmt.Errorf("no media storage configured") } p, err = t.mediaLoader.LoadPath(ref.ID) if err != nil { return "", "", fmt.Errorf("audio file not found: %v", err) } loadedLegacy = true } if loadedLegacy { p, err = resolveLoadedMediaRefPath(ctx, t.mediaLoader, p, "audio") } else { p, err = resolveStructuredMediaRefPath(ctx, p, "audio") } if err != nil { return "", "", err } mime = ref.MimeType if mime == "" || mime == "application/octet-stream" { mime = mimeFromAudioExt(filepath.Ext(p)) } return p, mime, nil } // callProvider dispatches audio analysis to the appropriate provider API. // Gemini: uses File API (upload → poll → file_data in generateContent). // OpenAI: uses input_audio content part in chat completions. // Transcription models: use the OpenAI-compatible audio transcription endpoint. func (t *ReadAudioTool) callProvider(ctx context.Context, cp credentialProvider, providerName, model string, params map[string]any) ([]byte, *providers.Usage, error) { prompt := GetParamString(params, "prompt", "Analyze this audio and describe its contents.") data, _ := params["data"].([]byte) mime := GetParamString(params, "mime", "audio/mpeg") audioPath, _ := params[audioLocalPathParam].(string) // Provider-specific paths require API credentials. Fail-fast (no silent // fallback to chat/completions) for any path we know won't work without // keys: gemini File API, openai input_audio, and any transcription-named // model under any ptype (e.g. openai_compat → DashScope qwen-audio). ptype := GetParamString(params, "_provider_type", providerTypeFromName(providerName)) if cp == nil && (ptype == "gemini" || ptype == "openai" || isTranscriptionModel(model)) { return nil, nil, fmt.Errorf("read_audio: provider %q requires API credentials for model %q", providerName, model) } if cp != nil { // Transcription models always go to /v1/audio/transcriptions regardless // of ptype — orthogonal to chat-vs-input_audio routing. Covers both // native openai (whisper-1, gpt-4o-transcribe) and openai_compat // providers exposing a /v1/audio/transcriptions endpoint. if isTranscriptionModel(model) { slog.Info("read_audio: using openai transcription API", "provider", providerName, "model", model, "size", len(data), "mime", mime) chatReq := providers.ChatRequest{ Messages: []providers.Message{{Role: "user", Content: prompt}}, Model: model, Options: map[string]any{"max_tokens": 16384}, } reservation, reserveErr := reserveToolLLMUsageWithMedia(ctx, t.usageCaps, t.Name(), providerName, model, chatReq, mediabudget.Payload{Kind: mediabudget.KindAudio, MIME: mime, Size: int64(len(data)), Path: audioPath}) if reserveErr != nil { return nil, nil, reserveErr } resp, err := openaiTranscriptionCall(ctx, cp.APIKey(), cp.APIBase(), model, data, mime) if reservation != nil { reservation.Reconcile(ctx, resp, err) } if err != nil { return nil, nil, fmt.Errorf("openai transcription call: %w", err) } return []byte(resp.Content), resp.Usage, nil } // Gemini: use File API (inlineData doesn't work for audio). if ptype == "gemini" { slog.Info("read_audio: using gemini file API", "provider", providerName, "model", model, "size", len(data), "mime", mime) chatReq := providers.ChatRequest{ Messages: []providers.Message{{Role: "user", Content: prompt}}, Model: model, Options: map[string]any{"max_tokens": 16384}, } reservation, reserveErr := reserveToolLLMUsageWithMedia(ctx, t.usageCaps, t.Name(), providerName, model, chatReq, mediabudget.Payload{Kind: mediabudget.KindAudio, MIME: mime, Size: int64(len(data)), Path: audioPath}) if reserveErr != nil { return nil, nil, reserveErr } resp, err := geminiFileAPICall(ctx, cp.APIKey(), model, prompt, data, mime, 120*time.Second) if reservation != nil { reservation.Reconcile(ctx, resp, err) } if err != nil { return nil, nil, fmt.Errorf("gemini file API: %w", err) } return []byte(resp.Content), resp.Usage, nil } // Native OpenAI chat-audio (gpt-4o-audio-preview etc.): input_audio content part. if ptype == "openai" { slog.Info("read_audio: using openai input_audio API", "provider", providerName, "model", model, "size", len(data), "mime", mime) chatReq := providers.ChatRequest{ Messages: []providers.Message{{Role: "user", Content: prompt}}, Model: model, Options: map[string]any{"max_tokens": 16384}, } reservation, reserveErr := reserveToolLLMUsageWithMedia(ctx, t.usageCaps, t.Name(), providerName, model, chatReq, mediabudget.Payload{Kind: mediabudget.KindAudio, MIME: mime, Size: int64(len(data)), Path: audioPath}) if reserveErr != nil { return nil, nil, reserveErr } resp, err := openaiAudioCall(ctx, cp.APIKey(), cp.APIBase(), model, prompt, data, mime) if reservation != nil { reservation.Reconcile(ctx, resp, err) } if err != nil { return nil, nil, fmt.Errorf("openai audio call: %w", err) } return []byte(resp.Content), resp.Usage, nil } } return nil, nil, fmt.Errorf("read_audio: unsupported audio route for provider %q (type %q) model %q; supported routes are Gemini File API, native OpenAI input_audio, or OpenAI-compatible transcription models", providerName, ptype, model) } // openaiAudioCall sends audio to OpenAI using the input_audio content part. func openaiAudioCall(ctx context.Context, apiKey, baseURL, model, prompt string, data []byte, mime string) (*providers.ChatResponse, error) { // Determine format from MIME (OpenAI supports: wav, mp3). format := "mp3" switch { case strings.Contains(mime, "wav"): format = "wav" case strings.Contains(mime, "mp3"), strings.Contains(mime, "mpeg"): format = "mp3" } b64 := base64.StdEncoding.EncodeToString(data) body := map[string]any{ "model": model, "messages": []map[string]any{ { "role": "user", "content": []map[string]any{ {"type": "text", "text": prompt}, {"type": "input_audio", "input_audio": map[string]string{ "data": b64, "format": format, }}, }, }, }, "max_tokens": 16384, } return callOpenAICompatJSON(ctx, apiKey, baseURL, body, 120*time.Second) }