diff --git a/cmd/gateway.go b/cmd/gateway.go index 559b8646..7b57e93f 100644 --- a/cmd/gateway.go +++ b/cmd/gateway.go @@ -189,6 +189,10 @@ func runGateway() { toolsReg.Register(tools.NewReadImageTool(providerRegistry)) toolsReg.Register(tools.NewCreateImageTool(providerRegistry)) + // Audio generation tool (MiniMax music + ElevenLabs sound effects) + toolsReg.Register(tools.NewCreateAudioTool(providerRegistry, + cfg.Tts.ElevenLabs.APIKey, cfg.Tts.ElevenLabs.BaseURL)) + // TTS (text-to-speech) system ttsMgr := setupTTS(cfg) if ttsMgr != nil { @@ -313,9 +317,11 @@ func runGateway() { // Block exec from accessing sensitive directories (data dir, .goclaw, config file). // Prevents `cp /app/data/config.json workspace/` and similar exfiltration. + // Exception: .goclaw/skills-store/ is allowed (skills may contain executable scripts). if execTool, ok := toolsReg.Get("exec"); ok { if et, ok := execTool.(*tools.ExecTool); ok { et.DenyPaths(dataDir, ".goclaw/") + et.AllowPathExemptions(".goclaw/skills-store/") if cfgPath := os.Getenv("GOCLAW_CONFIG"); cfgPath != "" { et.DenyPaths(cfgPath) } diff --git a/cmd/gateway_builtin_tools.go b/cmd/gateway_builtin_tools.go index e7b96753..c0deab99 100644 --- a/cmd/gateway_builtin_tools.go +++ b/cmd/gateway_builtin_tools.go @@ -63,6 +63,10 @@ func builtinToolSeedData() []store.BuiltinToolDef { Settings: json.RawMessage(`{"provider":"gemini","model":"veo-3.0-generate-preview"}`), Requires: []string{"video_gen_provider"}, }, + {Name: "create_audio", DisplayName: "Create Audio", Description: "Generate music or sound effects from text descriptions using AI", Category: "media", Enabled: true, + Settings: json.RawMessage(`{"provider":"minimax","model":"music-2.5+"}`), + Requires: []string{"audio_gen_provider"}, + }, {Name: "tts", DisplayName: "Text to Speech", Description: "Convert text to speech audio", Category: "media", Enabled: true, Requires: []string{"tts_provider"}, Metadata: json.RawMessage(`{"config_hint":"Config → TTS"}`), diff --git a/cmd/gateway_providers.go b/cmd/gateway_providers.go index c81fee76..238cdbb7 100644 --- a/cmd/gateway_providers.go +++ b/cmd/gateway_providers.go @@ -185,6 +185,14 @@ func registerProvidersFromDB(registry *providers.Registry, provStore store.Provi base = "https://coding-intl.dashscope.aliyuncs.com/v1" } registry.Register(providers.NewOpenAIProvider(p.Name, p.APIKey, base, "qwen3.5-plus")) + case store.ProviderSuno: + // Suno is a media-only provider (music gen). Register as OpenAI-compat + // so credentialProvider interface works for API key/base extraction. + base := p.APIBase + if base == "" { + base = "https://api.sunoapi.org" + } + registry.Register(providers.NewOpenAIProvider(p.Name, p.APIKey, base, "")) default: prov := providers.NewOpenAIProvider(p.Name, p.APIKey, p.APIBase, "") if p.ProviderType == store.ProviderMiniMax { diff --git a/internal/store/provider_store.go b/internal/store/provider_store.go index 60582729..9130d75a 100644 --- a/internal/store/provider_store.go +++ b/internal/store/provider_store.go @@ -24,6 +24,7 @@ const ( ProviderBailian = "bailian" ProviderChatGPTOAuth = "chatgpt_oauth" ProviderClaudeCLI = "claude_cli" + ProviderSuno = "suno" ) // ValidProviderTypes lists all accepted provider_type values. @@ -43,6 +44,7 @@ var ValidProviderTypes = map[string]bool{ ProviderBailian: true, ProviderChatGPTOAuth: true, ProviderClaudeCLI: true, + ProviderSuno: true, } // LLMProviderData represents an LLM provider configuration. diff --git a/internal/tools/create_audio.go b/internal/tools/create_audio.go new file mode 100644 index 00000000..182baedb --- /dev/null +++ b/internal/tools/create_audio.go @@ -0,0 +1,192 @@ +package tools + +import ( + "context" + "fmt" + "log/slog" + "os" + "path/filepath" + "time" + + "github.com/nextlevelbuilder/goclaw/internal/bus" + "github.com/nextlevelbuilder/goclaw/internal/providers" +) + +// audioGenProviderPriority is the default fallback order for music generation providers. +var audioGenProviderPriority = []string{"minimax"} + +// audioGenModelDefaults maps provider names to default audio generation models. +var audioGenModelDefaults = map[string]string{ + "minimax": "music-2.5+", +} + +// CreateAudioTool generates music or sound effects using AI audio generation APIs. +type CreateAudioTool struct { + registry *providers.Registry + elevenlabsAPIKey string + elevenlabsBaseURL string +} + +// NewCreateAudioTool creates a CreateAudioTool. +// elevenlabsKey and elevenlabsBase are used for ElevenLabs sound effects generation. +func NewCreateAudioTool(registry *providers.Registry, elevenlabsKey, elevenlabsBase string) *CreateAudioTool { + return &CreateAudioTool{ + registry: registry, + elevenlabsAPIKey: elevenlabsKey, + elevenlabsBaseURL: elevenlabsBase, + } +} + +func (t *CreateAudioTool) Name() string { return "create_audio" } + +func (t *CreateAudioTool) Description() string { + return "Generate music, sound effects, or ambient audio from a text description. Returns a MEDIA: path to the generated audio file." +} + +func (t *CreateAudioTool) Parameters() map[string]interface{} { + return map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "prompt": map[string]interface{}{ + "type": "string", + "description": "Description of audio to generate.", + }, + "type": map[string]interface{}{ + "type": "string", + "description": "Type of audio: 'music' (default) or 'sound_effect'.", + }, + "duration": map[string]interface{}{ + "type": "integer", + "description": "Duration in seconds.", + }, + "lyrics": map[string]interface{}{ + "type": "string", + "description": "Lyrics for music generation. Use [Verse], [Chorus] tags.", + }, + "instrumental": map[string]interface{}{ + "type": "boolean", + "description": "No vocals (default: false).", + }, + "provider": map[string]interface{}{ + "type": "string", + "description": "Force a specific provider (e.g. 'minimax').", + }, + }, + "required": []string{"prompt"}, + } +} + +func (t *CreateAudioTool) Execute(ctx context.Context, args map[string]interface{}) *Result { + prompt, _ := args["prompt"].(string) + if prompt == "" { + return ErrorResult("prompt is required") + } + + audioType, _ := args["type"].(string) + if audioType == "" { + audioType = "music" + } + + duration := 0 + if d, ok := args["duration"].(float64); ok { + duration = int(d) + } + + lyrics, _ := args["lyrics"].(string) + instrumental, _ := args["instrumental"].(bool) + forceProvider, _ := args["provider"].(string) + + var audioBytes []byte + var usage *providers.Usage + var providerName, model string + + // Sound effects: route directly to ElevenLabs if credentials available. + if audioType == "sound_effect" { + if t.elevenlabsAPIKey == "" { + return ErrorResult("sound_effect generation requires ElevenLabs credentials (elevenlabs.api_key in config)") + } + var err error + audioBytes, err = t.callElevenLabsSoundEffect(ctx, prompt, duration) + if err != nil { + return ErrorResult(fmt.Sprintf("ElevenLabs sound effect generation failed: %v", err)) + } + providerName = "elevenlabs" + model = "sound-generation" + } else { + // Music: use the provider chain. + chain := ResolveMediaProviderChain(ctx, "create_audio", forceProvider, "", + audioGenProviderPriority, audioGenModelDefaults, t.registry) + + // Inject prompt and music-specific params into each chain entry. + for i := range chain { + if chain[i].Params == nil { + chain[i].Params = make(map[string]any) + } + chain[i].Params["prompt"] = prompt + chain[i].Params["lyrics"] = lyrics + chain[i].Params["instrumental"] = instrumental + if duration > 0 { + chain[i].Params["duration"] = duration + } + } + + // Override timeout to 300s for music generation (slow API). + for i := range chain { + if chain[i].Timeout < 300 { + chain[i].Timeout = 300 + } + } + + chainResult, err := ExecuteWithChain(ctx, chain, t.registry, t.callProvider) + if err != nil { + return ErrorResult(fmt.Sprintf("audio generation failed: %v", err)) + } + audioBytes = chainResult.Data + usage = chainResult.Usage + providerName = chainResult.Provider + model = chainResult.Model + } + + // Save to workspace under date-based folder. + workspace := ToolWorkspaceFromCtx(ctx) + if workspace == "" { + workspace = os.TempDir() + } + dateDir := filepath.Join(workspace, "generated", time.Now().Format("2006-01-02")) + if err := os.MkdirAll(dateDir, 0755); err != nil { + return ErrorResult(fmt.Sprintf("failed to create output directory: %v", err)) + } + audioPath := filepath.Join(dateDir, fmt.Sprintf("goclaw_gen_%d.mp3", time.Now().UnixNano())) + if err := os.WriteFile(audioPath, audioBytes, 0644); err != nil { + return ErrorResult(fmt.Sprintf("failed to save generated audio: %v", err)) + } + + slog.Info("create_audio: audio saved", "path", audioPath, "provider", providerName, "type", audioType) + + result := &Result{ForLLM: fmt.Sprintf("MEDIA:%s", audioPath)} + result.Media = []bus.MediaFile{{Path: audioPath, MimeType: "audio/mpeg"}} + result.Deliverable = fmt.Sprintf("[Generated audio: %s]\nPrompt: %s", filepath.Base(audioPath), prompt) + result.Provider = providerName + result.Model = model + if usage != nil { + result.Usage = usage + } + return result +} + +// callProvider dispatches to the correct music generation implementation based on provider type. +func (t *CreateAudioTool) callProvider(ctx context.Context, cp credentialProvider, providerName, model string, params map[string]any) ([]byte, *providers.Usage, error) { + prompt := GetParamString(params, "prompt", "") + + slog.Info("create_audio: calling music generation API", + "provider", providerName, "model", model) + + switch ProviderTypeFromName(providerName) { + case "minimax": + return callMinimaxMusicGen(ctx, cp.APIKey(), cp.APIBase(), model, prompt, params) + case "suno": + return callSunoMusicGen(ctx, cp.APIKey(), cp.APIBase(), model, prompt, params) + default: + return nil, nil, fmt.Errorf("unsupported provider for audio generation: %q", providerName) + } +} diff --git a/internal/tools/create_audio_elevenlabs.go b/internal/tools/create_audio_elevenlabs.go new file mode 100644 index 00000000..2ec41c8b --- /dev/null +++ b/internal/tools/create_audio_elevenlabs.go @@ -0,0 +1,72 @@ +package tools + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// callElevenLabsSoundEffect generates a sound effect via the ElevenLabs sound generation API. +// Endpoint: POST {baseURL}/v1/sound-generation +// Auth: xi-api-key header (NOT Bearer token). +// Response: binary audio data directly (not JSON). +func (t *CreateAudioTool) callElevenLabsSoundEffect(ctx context.Context, prompt string, durationSeconds int) ([]byte, error) { + baseURL := t.elevenlabsBaseURL + if baseURL == "" { + baseURL = "https://api.elevenlabs.io" + } + + // Cap duration at ElevenLabs max of 30 seconds. + if durationSeconds > 30 { + durationSeconds = 30 + } + + body := map[string]interface{}{ + "text": prompt, + "output_format": "mp3_44100_128", + "prompt_influence": 0.3, + } + if durationSeconds > 0 { + body["duration_seconds"] = durationSeconds + } + + jsonBody, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("marshal request: %w", err) + } + + url := strings.TrimRight(baseURL, "/") + "/v1/sound-generation" + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonBody)) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("xi-api-key", t.elevenlabsAPIKey) + + client := &http.Client{Timeout: 60 * time.Second} + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("http request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + errBody, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("ElevenLabs API error %d: %s", resp.StatusCode, truncateBytes(errBody, 500)) + } + + audioBytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("read audio response: %w", err) + } + if len(audioBytes) == 0 { + return nil, fmt.Errorf("empty audio response from ElevenLabs") + } + + return audioBytes, nil +} diff --git a/internal/tools/create_audio_minimax.go b/internal/tools/create_audio_minimax.go new file mode 100644 index 00000000..cee76fbf --- /dev/null +++ b/internal/tools/create_audio_minimax.go @@ -0,0 +1,124 @@ +package tools + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + "github.com/nextlevelbuilder/goclaw/internal/providers" +) + +// callMinimaxMusicGen calls the MiniMax music generation API. +// Endpoint: POST {apiBase}/music_generation +// Response: data.audio or data.music (URL to audio file, downloaded and returned as bytes). +func callMinimaxMusicGen(ctx context.Context, apiKey, apiBase, model, prompt string, params map[string]any) ([]byte, *providers.Usage, error) { + lyrics := GetParamString(params, "lyrics", "") + instrumental := GetParamBool(params, "instrumental", false) + lyricsOptimizer := GetParamBool(params, "lyrics_optimizer", false) + sampleRate := GetParamString(params, "sample_rate", "44100") + bitrate := GetParamString(params, "bitrate", "320k") + + body := map[string]interface{}{ + "model": model, + "prompt": prompt, + "is_instrumental": instrumental, + "lyrics_optimizer": lyricsOptimizer, + "output_format": "url", + "audio_setting": map[string]interface{}{ + "sample_rate": sampleRate, + "bitrate": bitrate, + "format": "mp3", + }, + } + if lyrics != "" { + body["lyrics"] = lyrics + } + + jsonBody, err := json.Marshal(body) + if err != nil { + return nil, nil, fmt.Errorf("marshal request: %w", err) + } + + url := strings.TrimRight(apiBase, "/") + "/music_generation" + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonBody)) + if err != nil { + return nil, nil, fmt.Errorf("create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+apiKey) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return nil, nil, fmt.Errorf("http request: %w", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, nil, fmt.Errorf("read response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, nil, fmt.Errorf("API error %d: %s", resp.StatusCode, truncateBytes(respBody, 500)) + } + + var minimaxResp struct { + Data *struct { + Audio string `json:"audio"` + Music string `json:"music"` + } `json:"data"` + BaseResp *struct { + StatusCode int `json:"status_code"` + StatusMsg string `json:"status_msg"` + } `json:"base_resp"` + } + if err := json.Unmarshal(respBody, &minimaxResp); err != nil { + return nil, nil, fmt.Errorf("parse response: %w", err) + } + + if minimaxResp.BaseResp != nil && minimaxResp.BaseResp.StatusCode != 0 { + return nil, nil, fmt.Errorf("MiniMax API error %d: %s", + minimaxResp.BaseResp.StatusCode, minimaxResp.BaseResp.StatusMsg) + } + + if minimaxResp.Data == nil { + return nil, nil, fmt.Errorf("no data in MiniMax music response") + } + + audioURL := minimaxResp.Data.Audio + if audioURL == "" { + audioURL = minimaxResp.Data.Music + } + if audioURL == "" { + return nil, nil, fmt.Errorf("no audio URL in MiniMax music response") + } + + // Download the audio file from the returned URL. + dlReq, err := http.NewRequestWithContext(ctx, "GET", audioURL, nil) + if err != nil { + return nil, nil, fmt.Errorf("create download request: %w", err) + } + + dlClient := &http.Client{} + dlResp, err := dlClient.Do(dlReq) + if err != nil { + return nil, nil, fmt.Errorf("download audio: %w", err) + } + defer dlResp.Body.Close() + + if dlResp.StatusCode != http.StatusOK { + dlBody, _ := io.ReadAll(dlResp.Body) + return nil, nil, fmt.Errorf("download error %d: %s", dlResp.StatusCode, truncateBytes(dlBody, 300)) + } + + audioBytes, err := limitedReadAll(dlResp.Body, maxMediaDownloadBytes) + if err != nil { + return nil, nil, fmt.Errorf("read audio data: %w", err) + } + + return audioBytes, nil, nil +} diff --git a/internal/tools/create_audio_suno.go b/internal/tools/create_audio_suno.go new file mode 100644 index 00000000..399be2c6 --- /dev/null +++ b/internal/tools/create_audio_suno.go @@ -0,0 +1,15 @@ +package tools + +import ( + "context" + "fmt" + + "github.com/nextlevelbuilder/goclaw/internal/providers" +) + +// callSunoMusicGen generates music via the Suno API. +// Suno must be registered as an LLM provider with provider_type "suno". +func callSunoMusicGen(ctx context.Context, apiKey, apiBase, model, prompt string, params map[string]any) ([]byte, *providers.Usage, error) { + // TODO: Implement Suno music generation + return nil, nil, fmt.Errorf("suno music generation not yet implemented") +}