feat(tools): add create_audio tool for music and sound effects

New tool supporting MiniMax music generation, ElevenLabs sound effects,
and Suno music (stub). Registers Suno as a valid provider type.

- create_audio.go: tool definition with provider chain for music, direct ElevenLabs for SFX
- create_audio_minimax.go: MiniMax music API (/music_generation)
- create_audio_elevenlabs.go: ElevenLabs sound effects API (/v1/sound-generation)
- create_audio_suno.go: stub for future Suno integration
- Add ProviderSuno type to store and gateway registration
This commit is contained in:
viettranx committed 2026-03-08 20:10:00 +07:00
1 parent 5815437f78
commit 6345df3136
8 files changed
+423

No files matched your search

+6
View File
@@ -189,6 +189,10 @@ func runGateway() {
toolsReg.Register(tools.NewReadImageTool(providerRegistry))
toolsReg.Register(tools.NewCreateImageTool(providerRegistry))
// Audio generation tool (MiniMax music + ElevenLabs sound effects)
toolsReg.Register(tools.NewCreateAudioTool(providerRegistry,
cfg.Tts.ElevenLabs.APIKey, cfg.Tts.ElevenLabs.BaseURL))
// TTS (text-to-speech) system
ttsMgr := setupTTS(cfg)
if ttsMgr != nil {
@@ -313,9 +317,11 @@ func runGateway() {
// Block exec from accessing sensitive directories (data dir, .goclaw, config file).
// Prevents `cp /app/data/config.json workspace/` and similar exfiltration.
// Exception: .goclaw/skills-store/ is allowed (skills may contain executable scripts).
if execTool, ok := toolsReg.Get("exec"); ok {
if et, ok := execTool.(*tools.ExecTool); ok {
et.DenyPaths(dataDir, ".goclaw/")
et.AllowPathExemptions(".goclaw/skills-store/")
if cfgPath := os.Getenv("GOCLAW_CONFIG"); cfgPath != "" {
et.DenyPaths(cfgPath)
}
+4
View File
@@ -63,6 +63,10 @@ func builtinToolSeedData() []store.BuiltinToolDef {
Settings: json.RawMessage(`{"provider":"gemini","model":"veo-3.0-generate-preview"}`),
Requires: []string{"video_gen_provider"},
},
{Name: "create_audio", DisplayName: "Create Audio", Description: "Generate music or sound effects from text descriptions using AI", Category: "media", Enabled: true,
Settings: json.RawMessage(`{"provider":"minimax","model":"music-2.5+"}`),
Requires: []string{"audio_gen_provider"},
},
{Name: "tts", DisplayName: "Text to Speech", Description: "Convert text to speech audio", Category: "media", Enabled: true,
Requires: []string{"tts_provider"},
Metadata: json.RawMessage(`{"config_hint":"Config → TTS"}`),
+8
View File
@@ -185,6 +185,14 @@ func registerProvidersFromDB(registry *providers.Registry, provStore store.Provi
base = "https://coding-intl.dashscope.aliyuncs.com/v1"
}
registry.Register(providers.NewOpenAIProvider(p.Name, p.APIKey, base, "qwen3.5-plus"))
case store.ProviderSuno:
// Suno is a media-only provider (music gen). Register as OpenAI-compat
// so credentialProvider interface works for API key/base extraction.
base := p.APIBase
if base == "" {
base = "https://api.sunoapi.org"
}
registry.Register(providers.NewOpenAIProvider(p.Name, p.APIKey, base, ""))
default:
prov := providers.NewOpenAIProvider(p.Name, p.APIKey, p.APIBase, "")
if p.ProviderType == store.ProviderMiniMax {
+2
View File
@@ -24,6 +24,7 @@ const (
ProviderBailian = "bailian"
ProviderChatGPTOAuth = "chatgpt_oauth"
ProviderClaudeCLI = "claude_cli"
ProviderSuno = "suno"
)
// ValidProviderTypes lists all accepted provider_type values.
@@ -43,6 +44,7 @@ var ValidProviderTypes = map[string]bool{
ProviderBailian: true,
ProviderChatGPTOAuth: true,
ProviderClaudeCLI: true,
ProviderSuno: true,
}
// LLMProviderData represents an LLM provider configuration.
+192
View File
@@ -0,0 +1,192 @@
package tools
import (
"context"
"fmt"
"log/slog"
"os"
"path/filepath"
"time"
"github.com/nextlevelbuilder/goclaw/internal/bus"
"github.com/nextlevelbuilder/goclaw/internal/providers"
)
// audioGenProviderPriority is the default fallback order for music generation providers.
var audioGenProviderPriority = []string{"minimax"}
// audioGenModelDefaults maps provider names to default audio generation models.
var audioGenModelDefaults = map[string]string{
"minimax": "music-2.5+",
}
// CreateAudioTool generates music or sound effects using AI audio generation APIs.
type CreateAudioTool struct {
registry *providers.Registry
elevenlabsAPIKey string
elevenlabsBaseURL string
}
// NewCreateAudioTool creates a CreateAudioTool.
// elevenlabsKey and elevenlabsBase are used for ElevenLabs sound effects generation.
func NewCreateAudioTool(registry *providers.Registry, elevenlabsKey, elevenlabsBase string) *CreateAudioTool {
return &CreateAudioTool{
registry: registry,
elevenlabsAPIKey: elevenlabsKey,
elevenlabsBaseURL: elevenlabsBase,
}
}
func (t *CreateAudioTool) Name() string { return "create_audio" }
func (t *CreateAudioTool) Description() string {
return "Generate music, sound effects, or ambient audio from a text description. Returns a MEDIA: path to the generated audio file."
}
func (t *CreateAudioTool) Parameters() map[string]interface{} {
return map[string]interface{}{
"type": "object",
"properties": map[string]interface{}{
"prompt": map[string]interface{}{
"type": "string",
"description": "Description of audio to generate.",
},
"type": map[string]interface{}{
"type": "string",
"description": "Type of audio: 'music' (default) or 'sound_effect'.",
},
"duration": map[string]interface{}{
"type": "integer",
"description": "Duration in seconds.",
},
"lyrics": map[string]interface{}{
"type": "string",
"description": "Lyrics for music generation. Use [Verse], [Chorus] tags.",
},
"instrumental": map[string]interface{}{
"type": "boolean",
"description": "No vocals (default: false).",
},
"provider": map[string]interface{}{
"type": "string",
"description": "Force a specific provider (e.g. 'minimax').",
},
},
"required": []string{"prompt"},
}
}
func (t *CreateAudioTool) Execute(ctx context.Context, args map[string]interface{}) *Result {
prompt, _ := args["prompt"].(string)
if prompt == "" {
return ErrorResult("prompt is required")
}
audioType, _ := args["type"].(string)
if audioType == "" {
audioType = "music"
}
duration := 0
if d, ok := args["duration"].(float64); ok {
duration = int(d)
}
lyrics, _ := args["lyrics"].(string)
instrumental, _ := args["instrumental"].(bool)
forceProvider, _ := args["provider"].(string)
var audioBytes []byte
var usage *providers.Usage
var providerName, model string
// Sound effects: route directly to ElevenLabs if credentials available.
if audioType == "sound_effect" {
if t.elevenlabsAPIKey == "" {
return ErrorResult("sound_effect generation requires ElevenLabs credentials (elevenlabs.api_key in config)")
}
var err error
audioBytes, err = t.callElevenLabsSoundEffect(ctx, prompt, duration)
if err != nil {
return ErrorResult(fmt.Sprintf("ElevenLabs sound effect generation failed: %v", err))
}
providerName = "elevenlabs"
model = "sound-generation"
} else {
// Music: use the provider chain.
chain := ResolveMediaProviderChain(ctx, "create_audio", forceProvider, "",
audioGenProviderPriority, audioGenModelDefaults, t.registry)
// Inject prompt and music-specific params into each chain entry.
for i := range chain {
if chain[i].Params == nil {
chain[i].Params = make(map[string]any)
}
chain[i].Params["prompt"] = prompt
chain[i].Params["lyrics"] = lyrics
chain[i].Params["instrumental"] = instrumental
if duration > 0 {
chain[i].Params["duration"] = duration
}
}
// Override timeout to 300s for music generation (slow API).
for i := range chain {
if chain[i].Timeout < 300 {
chain[i].Timeout = 300
}
}
chainResult, err := ExecuteWithChain(ctx, chain, t.registry, t.callProvider)
if err != nil {
return ErrorResult(fmt.Sprintf("audio generation failed: %v", err))
}
audioBytes = chainResult.Data
usage = chainResult.Usage
providerName = chainResult.Provider
model = chainResult.Model
}
// Save to workspace under date-based folder.
workspace := ToolWorkspaceFromCtx(ctx)
if workspace == "" {
workspace = os.TempDir()
}
dateDir := filepath.Join(workspace, "generated", time.Now().Format("2006-01-02"))
if err := os.MkdirAll(dateDir, 0755); err != nil {
return ErrorResult(fmt.Sprintf("failed to create output directory: %v", err))
}
audioPath := filepath.Join(dateDir, fmt.Sprintf("goclaw_gen_%d.mp3", time.Now().UnixNano()))
if err := os.WriteFile(audioPath, audioBytes, 0644); err != nil {
return ErrorResult(fmt.Sprintf("failed to save generated audio: %v", err))
}
slog.Info("create_audio: audio saved", "path", audioPath, "provider", providerName, "type", audioType)
result := &Result{ForLLM: fmt.Sprintf("MEDIA:%s", audioPath)}
result.Media = []bus.MediaFile{{Path: audioPath, MimeType: "audio/mpeg"}}
result.Deliverable = fmt.Sprintf("[Generated audio: %s]\nPrompt: %s", filepath.Base(audioPath), prompt)
result.Provider = providerName
result.Model = model
if usage != nil {
result.Usage = usage
}
return result
}
// callProvider dispatches to the correct music generation implementation based on provider type.
func (t *CreateAudioTool) callProvider(ctx context.Context, cp credentialProvider, providerName, model string, params map[string]any) ([]byte, *providers.Usage, error) {
prompt := GetParamString(params, "prompt", "")
slog.Info("create_audio: calling music generation API",
"provider", providerName, "model", model)
switch ProviderTypeFromName(providerName) {
case "minimax":
return callMinimaxMusicGen(ctx, cp.APIKey(), cp.APIBase(), model, prompt, params)
case "suno":
return callSunoMusicGen(ctx, cp.APIKey(), cp.APIBase(), model, prompt, params)
default:
return nil, nil, fmt.Errorf("unsupported provider for audio generation: %q", providerName)
}
}
+72
View File
@@ -0,0 +1,72 @@
package tools
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
)
// callElevenLabsSoundEffect generates a sound effect via the ElevenLabs sound generation API.
// Endpoint: POST {baseURL}/v1/sound-generation
// Auth: xi-api-key header (NOT Bearer token).
// Response: binary audio data directly (not JSON).
func (t *CreateAudioTool) callElevenLabsSoundEffect(ctx context.Context, prompt string, durationSeconds int) ([]byte, error) {
baseURL := t.elevenlabsBaseURL
if baseURL == "" {
baseURL = "https://api.elevenlabs.io"
}
// Cap duration at ElevenLabs max of 30 seconds.
if durationSeconds > 30 {
durationSeconds = 30
}
body := map[string]interface{}{
"text": prompt,
"output_format": "mp3_44100_128",
"prompt_influence": 0.3,
}
if durationSeconds > 0 {
body["duration_seconds"] = durationSeconds
}
jsonBody, err := json.Marshal(body)
if err != nil {
return nil, fmt.Errorf("marshal request: %w", err)
}
url := strings.TrimRight(baseURL, "/") + "/v1/sound-generation"
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonBody))
if err != nil {
return nil, fmt.Errorf("create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("xi-api-key", t.elevenlabsAPIKey)
client := &http.Client{Timeout: 60 * time.Second}
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("http request: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
errBody, _ := io.ReadAll(resp.Body)
return nil, fmt.Errorf("ElevenLabs API error %d: %s", resp.StatusCode, truncateBytes(errBody, 500))
}
audioBytes, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("read audio response: %w", err)
}
if len(audioBytes) == 0 {
return nil, fmt.Errorf("empty audio response from ElevenLabs")
}
return audioBytes, nil
}
+124
View File
@@ -0,0 +1,124 @@
package tools
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"github.com/nextlevelbuilder/goclaw/internal/providers"
)
// callMinimaxMusicGen calls the MiniMax music generation API.
// Endpoint: POST {apiBase}/music_generation
// Response: data.audio or data.music (URL to audio file, downloaded and returned as bytes).
func callMinimaxMusicGen(ctx context.Context, apiKey, apiBase, model, prompt string, params map[string]any) ([]byte, *providers.Usage, error) {
lyrics := GetParamString(params, "lyrics", "")
instrumental := GetParamBool(params, "instrumental", false)
lyricsOptimizer := GetParamBool(params, "lyrics_optimizer", false)
sampleRate := GetParamString(params, "sample_rate", "44100")
bitrate := GetParamString(params, "bitrate", "320k")
body := map[string]interface{}{
"model": model,
"prompt": prompt,
"is_instrumental": instrumental,
"lyrics_optimizer": lyricsOptimizer,
"output_format": "url",
"audio_setting": map[string]interface{}{
"sample_rate": sampleRate,
"bitrate": bitrate,
"format": "mp3",
},
}
if lyrics != "" {
body["lyrics"] = lyrics
}
jsonBody, err := json.Marshal(body)
if err != nil {
return nil, nil, fmt.Errorf("marshal request: %w", err)
}
url := strings.TrimRight(apiBase, "/") + "/music_generation"
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonBody))
if err != nil {
return nil, nil, fmt.Errorf("create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+apiKey)
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return nil, nil, fmt.Errorf("http request: %w", err)
}
defer resp.Body.Close()
respBody, err := io.ReadAll(resp.Body)
if err != nil {
return nil, nil, fmt.Errorf("read response: %w", err)
}
if resp.StatusCode != http.StatusOK {
return nil, nil, fmt.Errorf("API error %d: %s", resp.StatusCode, truncateBytes(respBody, 500))
}
var minimaxResp struct {
Data *struct {
Audio string `json:"audio"`
Music string `json:"music"`
} `json:"data"`
BaseResp *struct {
StatusCode int `json:"status_code"`
StatusMsg string `json:"status_msg"`
} `json:"base_resp"`
}
if err := json.Unmarshal(respBody, &minimaxResp); err != nil {
return nil, nil, fmt.Errorf("parse response: %w", err)
}
if minimaxResp.BaseResp != nil && minimaxResp.BaseResp.StatusCode != 0 {
return nil, nil, fmt.Errorf("MiniMax API error %d: %s",
minimaxResp.BaseResp.StatusCode, minimaxResp.BaseResp.StatusMsg)
}
if minimaxResp.Data == nil {
return nil, nil, fmt.Errorf("no data in MiniMax music response")
}
audioURL := minimaxResp.Data.Audio
if audioURL == "" {
audioURL = minimaxResp.Data.Music
}
if audioURL == "" {
return nil, nil, fmt.Errorf("no audio URL in MiniMax music response")
}
// Download the audio file from the returned URL.
dlReq, err := http.NewRequestWithContext(ctx, "GET", audioURL, nil)
if err != nil {
return nil, nil, fmt.Errorf("create download request: %w", err)
}
dlClient := &http.Client{}
dlResp, err := dlClient.Do(dlReq)
if err != nil {
return nil, nil, fmt.Errorf("download audio: %w", err)
}
defer dlResp.Body.Close()
if dlResp.StatusCode != http.StatusOK {
dlBody, _ := io.ReadAll(dlResp.Body)
return nil, nil, fmt.Errorf("download error %d: %s", dlResp.StatusCode, truncateBytes(dlBody, 300))
}
audioBytes, err := limitedReadAll(dlResp.Body, maxMediaDownloadBytes)
if err != nil {
return nil, nil, fmt.Errorf("read audio data: %w", err)
}
return audioBytes, nil, nil
}
+15
View File
@@ -0,0 +1,15 @@
package tools
import (
"context"
"fmt"
"github.com/nextlevelbuilder/goclaw/internal/providers"
)
// callSunoMusicGen generates music via the Suno API.
// Suno must be registered as an LLM provider with provider_type "suno".
func callSunoMusicGen(ctx context.Context, apiKey, apiBase, model, prompt string, params map[string]any) ([]byte, *providers.Usage, error) {
// TODO: Implement Suno music generation
return nil, nil, fmt.Errorf("suno music generation not yet implemented")
}