diff --git a/cmd/gateway_agents.go b/cmd/gateway_agents.go index 205f7547..020ec993 100644 --- a/cmd/gateway_agents.go +++ b/cmd/gateway_agents.go @@ -4,6 +4,8 @@ import ( "context" "log/slog" + "github.com/nextlevelbuilder/goclaw/internal/audio/elevenlabs" + minimaxaudio "github.com/nextlevelbuilder/goclaw/internal/audio/minimax" "github.com/nextlevelbuilder/goclaw/internal/bus" "github.com/nextlevelbuilder/goclaw/internal/config" "github.com/nextlevelbuilder/goclaw/internal/memory" @@ -275,12 +277,42 @@ func setupTTS(cfg *config.Config) *tts.Manager { return mgr } -// setupAudioExtras wires optional STT + Music static defaults into the audio -// Manager. Phase 1 placeholder: body is empty — Phase 3 wires Music providers, -// Phase 4 wires STT providers. Kept here so the call-site in gateway_setup.go -// is stable from Phase 1 onward. -// -//nolint:unused // Phase 1 stub; wired in Phase 3/4. -func setupAudioExtras(_ *config.Config, _ *tts.Manager) { - // Intentionally empty. Phase 3/4 will register Music/STT providers here. +// setupAudioExtras wires Music and SFX providers into the audio Manager. +// ElevenLabs is registered for both SFX and Music when an API key is present. +// MiniMax music is registered when cfg.Audio.Music is configured with a key. +// Phase 4 will add STT providers here. +func setupAudioExtras(cfg *config.Config, mgr *tts.Manager) { + ellKey := cfg.Tts.ElevenLabs.APIKey + ellBase := cfg.Tts.ElevenLabs.BaseURL + + // ElevenLabs SFX — reuse TTS credentials. + if ellKey != "" { + mgr.RegisterSFX(elevenlabs.NewSFXProvider(elevenlabs.Config{ + APIKey: ellKey, + BaseURL: ellBase, + })) + slog.Info("audio.sfx: elevenlabs registered") + } + + // ElevenLabs Music — same credentials, uses /v1/music endpoint. + if ellKey != "" { + mgr.RegisterMusic(elevenlabs.NewMusicProvider(elevenlabs.Config{ + APIKey: ellKey, + BaseURL: ellBase, + })) + slog.Info("audio.music: elevenlabs registered") + } + + // MiniMax Music — optional, from cfg.Audio.Music block. + if cfg.Audio != nil && cfg.Audio.Music != nil { + mc := cfg.Audio.Music + if mc.APIKey != "" { + mgr.RegisterMusic(minimaxaudio.NewMusicProvider(minimaxaudio.MusicConfig{ + APIKey: mc.APIKey, + APIBase: mc.BaseURL, + Model: mc.Model, + })) + slog.Info("audio.music: minimax registered") + } + } } diff --git a/cmd/gateway_setup.go b/cmd/gateway_setup.go index bde81cc2..a527c4d7 100644 --- a/cmd/gateway_setup.go +++ b/cmd/gateway_setup.go @@ -133,17 +133,17 @@ func setupToolRegistry( toolsReg.Register(tools.NewReadImageTool(providerRegistry)) toolsReg.Register(tools.NewCreateImageTool(providerRegistry)) - // Audio generation tool (MiniMax music + ElevenLabs sound effects) - toolsReg.Register(tools.NewCreateAudioTool(providerRegistry, - cfg.Tts.ElevenLabs.APIKey, cfg.Tts.ElevenLabs.BaseURL)) - - // Audio system (TTS today, STT/Music/SFX in later phases) — always create - // TtsTool so config reload can populate it later. + // Audio system: build Manager first so Music/SFX providers are registered + // before the create_audio tool is constructed. ttsMgr := setupTTS(cfg) if ttsMgr == nil { ttsMgr = tts.NewManager(tts.ManagerConfig{}) } - setupAudioExtras(cfg, ttsMgr) // Phase 1 stub; Phase 3/4 wire Music/STT. + setupAudioExtras(cfg, ttsMgr) // Phase 3: registers Music + SFX providers. + + // Audio generation tool — backed by audio.Manager (Music + SFX). + toolsReg.Register(tools.NewCreateAudioTool(ttsMgr)) + ttsTool = tools.NewTtsTool(ttsMgr) toolsReg.Register(ttsTool) if ttsMgr.HasProviders() { diff --git a/internal/tools/create_audio.go b/internal/tools/create_audio.go index f7578d1a..a74bceb0 100644 --- a/internal/tools/create_audio.go +++ b/internal/tools/create_audio.go @@ -8,36 +8,22 @@ import ( "path/filepath" "time" + "github.com/nextlevelbuilder/goclaw/internal/audio" "github.com/nextlevelbuilder/goclaw/internal/bus" - "github.com/nextlevelbuilder/goclaw/internal/providers" ) -// audioGenProviderPriority is the default fallback order for music generation providers. -var audioGenProviderPriority = []string{"minimax"} - -// audioGenModelDefaults maps provider names to default audio generation models. -var audioGenModelDefaults = map[string]string{ - "minimax": "music-2.5+", -} - -// CreateAudioTool generates music or sound effects using AI audio generation APIs. +// CreateAudioTool generates music or sound effects using the audio.Manager. +// Provider selection and fallback are handled by the Manager's chain logic. type CreateAudioTool struct { - registry *providers.Registry - elevenlabsAPIKey string - elevenlabsBaseURL string - vaultIntc *VaultInterceptor + audioMgr *audio.Manager + vaultIntc *VaultInterceptor } func (t *CreateAudioTool) SetVaultInterceptor(v *VaultInterceptor) { t.vaultIntc = v } -// NewCreateAudioTool creates a CreateAudioTool. -// elevenlabsKey and elevenlabsBase are used for ElevenLabs sound effects generation. -func NewCreateAudioTool(registry *providers.Registry, elevenlabsKey, elevenlabsBase string) *CreateAudioTool { - return &CreateAudioTool{ - registry: registry, - elevenlabsAPIKey: elevenlabsKey, - elevenlabsBaseURL: elevenlabsBase, - } +// NewCreateAudioTool creates a CreateAudioTool backed by the audio.Manager. +func NewCreateAudioTool(audioMgr *audio.Manager) *CreateAudioTool { + return &CreateAudioTool{audioMgr: audioMgr} } func (t *CreateAudioTool) Name() string { return "create_audio" } @@ -70,10 +56,6 @@ func (t *CreateAudioTool) Parameters() map[string]any { "type": "boolean", "description": "No vocals (default: false).", }, - "provider": map[string]any{ - "type": "string", - "description": "Force a specific provider (e.g. 'minimax').", - }, "filename_hint": map[string]any{ "type": "string", "description": "Short descriptive filename (no extension). Example: 'epic-battle-theme', 'rain-ambience'.", @@ -101,57 +83,38 @@ func (t *CreateAudioTool) Execute(ctx context.Context, args map[string]any) *Res lyrics, _ := args["lyrics"].(string) instrumental, _ := args["instrumental"].(bool) - forceProvider, _ := args["provider"].(string) filenameHint, _ := args["filename_hint"].(string) - var audioBytes []byte - var usage *providers.Usage - var providerName, model string + var res *audio.AudioResult + var err error - // Sound effects: route directly to ElevenLabs if credentials available. - if audioType == "sound_effect" { - if t.elevenlabsAPIKey == "" { - return ErrorResult("sound_effect generation requires ElevenLabs credentials (elevenlabs.api_key in config)") - } - var err error - audioBytes, err = t.callElevenLabsSoundEffect(ctx, prompt, duration) + switch audioType { + case "sound_effect": + res, err = t.audioMgr.GenerateSFX(ctx, audio.SFXOptions{ + Prompt: prompt, + Duration: duration, + }) if err != nil { - return ErrorResult(fmt.Sprintf("ElevenLabs sound effect generation failed: %v", err)) + return ErrorResult(fmt.Sprintf("sound effect generation failed: %v", err)) } - providerName = "elevenlabs" - model = "sound-generation" - } else { - // Music: use the provider chain. - chain := ResolveMediaProviderChain(ctx, "create_audio", forceProvider, "", - audioGenProviderPriority, audioGenModelDefaults, t.registry) - - // Inject prompt and music-specific params into each chain entry. - for i := range chain { - if chain[i].Params == nil { - chain[i].Params = make(map[string]any) - } - chain[i].Params["prompt"] = prompt - chain[i].Params["lyrics"] = lyrics - chain[i].Params["instrumental"] = instrumental - } - - // Override timeout to 300s for music generation (slow API). - for i := range chain { - if chain[i].Timeout < 300 { - chain[i].Timeout = 300 - } - } - - chainResult, err := ExecuteWithChain(ctx, chain, t.registry, t.callProvider) + case "music": + res, err = t.audioMgr.GenerateMusic(ctx, audio.MusicOptions{ + Prompt: prompt, + Lyrics: lyrics, + Instrumental: instrumental, + TimeoutSec: 300, + }) if err != nil { - return ErrorResult(fmt.Sprintf("audio generation failed: %v", err)) + return ErrorResult(fmt.Sprintf("music generation failed: %v", err)) } - audioBytes = chainResult.Data - usage = chainResult.Usage - providerName = chainResult.Provider - model = chainResult.Model + default: + return ErrorResult(fmt.Sprintf("unsupported audio type: %s", audioType)) } + audioBytes := res.Audio + providerName := res.Provider + model := res.Model + // Save to workspace under date-based folder. workspace := ToolWorkspaceFromCtx(ctx) if workspace == "" { @@ -171,7 +134,9 @@ func (t *CreateAudioTool) Execute(ctx context.Context, args map[string]any) *Res slog.Warn("create_audio: file missing immediately after write", "path", audioPath, "error", err) return ErrorResult(fmt.Sprintf("generated audio file missing after write: %v", err)) } else { - slog.Info("create_audio: file saved", "path", audioPath, "size", fi.Size(), "data_len", len(audioBytes), "provider", providerName, "type", audioType) + slog.Info("create_audio: file saved", + "path", audioPath, "size", fi.Size(), + "data_len", len(audioBytes), "model", model, "type", audioType) } result := &Result{ForLLM: fmt.Sprintf("MEDIA:%s\nUse the EXACT filename when referencing: %s", audioPath, filepath.Base(audioPath))} @@ -182,28 +147,5 @@ func (t *CreateAudioTool) Execute(ctx context.Context, args map[string]any) *Res } result.Provider = providerName result.Model = model - if usage != nil { - result.Usage = usage - } return result } - -// callProvider dispatches to the correct music generation implementation based on provider type. -func (t *CreateAudioTool) callProvider(ctx context.Context, cp credentialProvider, providerName, model string, params map[string]any) ([]byte, *providers.Usage, error) { - if cp == nil { - return nil, nil, fmt.Errorf("provider %q does not expose API credentials required for audio generation", providerName) - } - prompt := GetParamString(params, "prompt", "") - - slog.Info("create_audio: calling music generation API", - "provider", providerName, "model", model) - - switch GetParamString(params, "_provider_type", providerTypeFromName(providerName)) { - case "minimax": - return callMinimaxMusicGen(ctx, cp.APIKey(), cp.APIBase(), model, prompt, params) - case "suno": - return callSunoMusicGen(ctx, cp.APIKey(), cp.APIBase(), model, prompt, params) - default: - return nil, nil, fmt.Errorf("unsupported provider for audio generation: %q", providerName) - } -} diff --git a/internal/tools/create_audio_elevenlabs.go b/internal/tools/create_audio_elevenlabs.go deleted file mode 100644 index e3a6fe45..00000000 --- a/internal/tools/create_audio_elevenlabs.go +++ /dev/null @@ -1,26 +0,0 @@ -package tools - -import ( - "context" - - "github.com/nextlevelbuilder/goclaw/internal/audio" - "github.com/nextlevelbuilder/goclaw/internal/audio/elevenlabs" -) - -// callElevenLabsSoundEffect delegates to audio/elevenlabs.SFXProvider. -// Phase 1 keeps this thin shim so create_audio.go callers don't change; -// Phase 3 removes the shim and wires audio.Manager.GenerateSFX directly. -func (t *CreateAudioTool) callElevenLabsSoundEffect(ctx context.Context, prompt string, durationSeconds int) ([]byte, error) { - provider := elevenlabs.NewSFXProvider(elevenlabs.Config{ - APIKey: t.elevenlabsAPIKey, - BaseURL: t.elevenlabsBaseURL, - }) - result, err := provider.GenerateSFX(ctx, audio.SFXOptions{ - Prompt: prompt, - Duration: durationSeconds, - }) - if err != nil { - return nil, err - } - return result.Audio, nil -} diff --git a/internal/tools/create_audio_minimax.go b/internal/tools/create_audio_minimax.go deleted file mode 100644 index 15f2fe8a..00000000 --- a/internal/tools/create_audio_minimax.go +++ /dev/null @@ -1,132 +0,0 @@ -package tools - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "strings" - - "github.com/nextlevelbuilder/goclaw/internal/providers" -) - -// callMinimaxMusicGen calls the MiniMax music generation API. -// Endpoint: POST {apiBase}/music_generation -// Response: data.audio or data.music (URL to audio file, downloaded and returned as bytes). -func callMinimaxMusicGen(ctx context.Context, apiKey, apiBase, model, prompt string, params map[string]any) ([]byte, *providers.Usage, error) { - lyrics := GetParamString(params, "lyrics", "") - instrumental := GetParamBool(params, "instrumental", false) - lyricsOptimizer := GetParamBool(params, "lyrics_optimizer", false) - sampleRate := GetParamInt(params, "sample_rate", 44100) - bitrate := GetParamInt(params, "bitrate", 256000) - - // lyrics is required when is_instrumental is false - if !instrumental && lyrics == "" { - instrumental = true - } - - // NOTE: MiniMax music API does NOT support a duration parameter. - // Audio length is determined by lyrics length (vocal) or the model's default (~3-4 min for instrumental). - - body := map[string]any{ - "model": model, - "prompt": prompt, - "is_instrumental": instrumental, - "lyrics_optimizer": lyricsOptimizer, - "output_format": "url", - "audio_setting": map[string]any{ - "sample_rate": sampleRate, - "bitrate": bitrate, - "format": "mp3", - }, - } - if lyrics != "" { - body["lyrics"] = lyrics - } - - jsonBody, err := json.Marshal(body) - if err != nil { - return nil, nil, fmt.Errorf("marshal request: %w", err) - } - - url := strings.TrimRight(apiBase, "/") + "/music_generation" - req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonBody)) - if err != nil { - return nil, nil, fmt.Errorf("create request: %w", err) - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", "Bearer "+apiKey) - - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - return nil, nil, fmt.Errorf("http request: %w", err) - } - defer resp.Body.Close() - - respBody, err := io.ReadAll(resp.Body) - if err != nil { - return nil, nil, fmt.Errorf("read response: %w", err) - } - if resp.StatusCode != http.StatusOK { - return nil, nil, fmt.Errorf("API error %d: %s", resp.StatusCode, truncateBytes(respBody, 500)) - } - - var minimaxResp struct { - Data *struct { - Audio string `json:"audio"` - Music string `json:"music"` - } `json:"data"` - BaseResp *struct { - StatusCode int `json:"status_code"` - StatusMsg string `json:"status_msg"` - } `json:"base_resp"` - } - if err := json.Unmarshal(respBody, &minimaxResp); err != nil { - return nil, nil, fmt.Errorf("parse response: %w", err) - } - - if minimaxResp.BaseResp != nil && minimaxResp.BaseResp.StatusCode != 0 { - return nil, nil, fmt.Errorf("MiniMax API error %d: %s", - minimaxResp.BaseResp.StatusCode, minimaxResp.BaseResp.StatusMsg) - } - - if minimaxResp.Data == nil { - return nil, nil, fmt.Errorf("no data in MiniMax music response") - } - - audioURL := minimaxResp.Data.Audio - if audioURL == "" { - audioURL = minimaxResp.Data.Music - } - if audioURL == "" { - return nil, nil, fmt.Errorf("no audio URL in MiniMax music response") - } - - // Download the audio file from the returned URL. - dlReq, err := http.NewRequestWithContext(ctx, "GET", audioURL, nil) - if err != nil { - return nil, nil, fmt.Errorf("create download request: %w", err) - } - - dlClient := &http.Client{} - dlResp, err := dlClient.Do(dlReq) - if err != nil { - return nil, nil, fmt.Errorf("download audio: %w", err) - } - defer dlResp.Body.Close() - - if dlResp.StatusCode != http.StatusOK { - dlBody, _ := io.ReadAll(dlResp.Body) - return nil, nil, fmt.Errorf("download error %d: %s", dlResp.StatusCode, truncateBytes(dlBody, 300)) - } - - audioBytes, err := limitedReadAll(dlResp.Body, maxMediaDownloadBytes) - if err != nil { - return nil, nil, fmt.Errorf("read audio data: %w", err) - } - - return audioBytes, nil, nil -} diff --git a/internal/tools/create_audio_suno.go b/internal/tools/create_audio_suno.go deleted file mode 100644 index 399be2c6..00000000 --- a/internal/tools/create_audio_suno.go +++ /dev/null @@ -1,15 +0,0 @@ -package tools - -import ( - "context" - "fmt" - - "github.com/nextlevelbuilder/goclaw/internal/providers" -) - -// callSunoMusicGen generates music via the Suno API. -// Suno must be registered as an LLM provider with provider_type "suno". -func callSunoMusicGen(ctx context.Context, apiKey, apiBase, model, prompt string, params map[string]any) ([]byte, *providers.Usage, error) { - // TODO: Implement Suno music generation - return nil, nil, fmt.Errorf("suno music generation not yet implemented") -}