mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-14 04:16:06 +00:00
feat(tools): add create_audio tool for music and sound effects
New tool supporting MiniMax music generation, ElevenLabs sound effects, and Suno music (stub). Registers Suno as a valid provider type. - create_audio.go: tool definition with provider chain for music, direct ElevenLabs for SFX - create_audio_minimax.go: MiniMax music API (/music_generation) - create_audio_elevenlabs.go: ElevenLabs sound effects API (/v1/sound-generation) - create_audio_suno.go: stub for future Suno integration - Add ProviderSuno type to store and gateway registration
This commit is contained in:
1 parent
5815437f78
commit
6345df3136
8 files changed
+423
No files matched your search
@@ -189,6 +189,10 @@ func runGateway() {
|
||||
toolsReg.Register(tools.NewReadImageTool(providerRegistry))
|
||||
toolsReg.Register(tools.NewCreateImageTool(providerRegistry))
|
||||
|
||||
// Audio generation tool (MiniMax music + ElevenLabs sound effects)
|
||||
toolsReg.Register(tools.NewCreateAudioTool(providerRegistry,
|
||||
cfg.Tts.ElevenLabs.APIKey, cfg.Tts.ElevenLabs.BaseURL))
|
||||
|
||||
// TTS (text-to-speech) system
|
||||
ttsMgr := setupTTS(cfg)
|
||||
if ttsMgr != nil {
|
||||
@@ -313,9 +317,11 @@ func runGateway() {
|
||||
|
||||
// Block exec from accessing sensitive directories (data dir, .goclaw, config file).
|
||||
// Prevents `cp /app/data/config.json workspace/` and similar exfiltration.
|
||||
// Exception: .goclaw/skills-store/ is allowed (skills may contain executable scripts).
|
||||
if execTool, ok := toolsReg.Get("exec"); ok {
|
||||
if et, ok := execTool.(*tools.ExecTool); ok {
|
||||
et.DenyPaths(dataDir, ".goclaw/")
|
||||
et.AllowPathExemptions(".goclaw/skills-store/")
|
||||
if cfgPath := os.Getenv("GOCLAW_CONFIG"); cfgPath != "" {
|
||||
et.DenyPaths(cfgPath)
|
||||
}
|
||||
|
||||
@@ -63,6 +63,10 @@ func builtinToolSeedData() []store.BuiltinToolDef {
|
||||
Settings: json.RawMessage(`{"provider":"gemini","model":"veo-3.0-generate-preview"}`),
|
||||
Requires: []string{"video_gen_provider"},
|
||||
},
|
||||
{Name: "create_audio", DisplayName: "Create Audio", Description: "Generate music or sound effects from text descriptions using AI", Category: "media", Enabled: true,
|
||||
Settings: json.RawMessage(`{"provider":"minimax","model":"music-2.5+"}`),
|
||||
Requires: []string{"audio_gen_provider"},
|
||||
},
|
||||
{Name: "tts", DisplayName: "Text to Speech", Description: "Convert text to speech audio", Category: "media", Enabled: true,
|
||||
Requires: []string{"tts_provider"},
|
||||
Metadata: json.RawMessage(`{"config_hint":"Config → TTS"}`),
|
||||
|
||||
@@ -185,6 +185,14 @@ func registerProvidersFromDB(registry *providers.Registry, provStore store.Provi
|
||||
base = "https://coding-intl.dashscope.aliyuncs.com/v1"
|
||||
}
|
||||
registry.Register(providers.NewOpenAIProvider(p.Name, p.APIKey, base, "qwen3.5-plus"))
|
||||
case store.ProviderSuno:
|
||||
// Suno is a media-only provider (music gen). Register as OpenAI-compat
|
||||
// so credentialProvider interface works for API key/base extraction.
|
||||
base := p.APIBase
|
||||
if base == "" {
|
||||
base = "https://api.sunoapi.org"
|
||||
}
|
||||
registry.Register(providers.NewOpenAIProvider(p.Name, p.APIKey, base, ""))
|
||||
default:
|
||||
prov := providers.NewOpenAIProvider(p.Name, p.APIKey, p.APIBase, "")
|
||||
if p.ProviderType == store.ProviderMiniMax {
|
||||
|
||||
@@ -24,6 +24,7 @@ const (
|
||||
ProviderBailian = "bailian"
|
||||
ProviderChatGPTOAuth = "chatgpt_oauth"
|
||||
ProviderClaudeCLI = "claude_cli"
|
||||
ProviderSuno = "suno"
|
||||
)
|
||||
|
||||
// ValidProviderTypes lists all accepted provider_type values.
|
||||
@@ -43,6 +44,7 @@ var ValidProviderTypes = map[string]bool{
|
||||
ProviderBailian: true,
|
||||
ProviderChatGPTOAuth: true,
|
||||
ProviderClaudeCLI: true,
|
||||
ProviderSuno: true,
|
||||
}
|
||||
|
||||
// LLMProviderData represents an LLM provider configuration.
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/bus"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/providers"
|
||||
)
|
||||
|
||||
// audioGenProviderPriority is the default fallback order for music generation providers.
|
||||
var audioGenProviderPriority = []string{"minimax"}
|
||||
|
||||
// audioGenModelDefaults maps provider names to default audio generation models.
|
||||
var audioGenModelDefaults = map[string]string{
|
||||
"minimax": "music-2.5+",
|
||||
}
|
||||
|
||||
// CreateAudioTool generates music or sound effects using AI audio generation APIs.
|
||||
type CreateAudioTool struct {
|
||||
registry *providers.Registry
|
||||
elevenlabsAPIKey string
|
||||
elevenlabsBaseURL string
|
||||
}
|
||||
|
||||
// NewCreateAudioTool creates a CreateAudioTool.
|
||||
// elevenlabsKey and elevenlabsBase are used for ElevenLabs sound effects generation.
|
||||
func NewCreateAudioTool(registry *providers.Registry, elevenlabsKey, elevenlabsBase string) *CreateAudioTool {
|
||||
return &CreateAudioTool{
|
||||
registry: registry,
|
||||
elevenlabsAPIKey: elevenlabsKey,
|
||||
elevenlabsBaseURL: elevenlabsBase,
|
||||
}
|
||||
}
|
||||
|
||||
func (t *CreateAudioTool) Name() string { return "create_audio" }
|
||||
|
||||
func (t *CreateAudioTool) Description() string {
|
||||
return "Generate music, sound effects, or ambient audio from a text description. Returns a MEDIA: path to the generated audio file."
|
||||
}
|
||||
|
||||
func (t *CreateAudioTool) Parameters() map[string]interface{} {
|
||||
return map[string]interface{}{
|
||||
"type": "object",
|
||||
"properties": map[string]interface{}{
|
||||
"prompt": map[string]interface{}{
|
||||
"type": "string",
|
||||
"description": "Description of audio to generate.",
|
||||
},
|
||||
"type": map[string]interface{}{
|
||||
"type": "string",
|
||||
"description": "Type of audio: 'music' (default) or 'sound_effect'.",
|
||||
},
|
||||
"duration": map[string]interface{}{
|
||||
"type": "integer",
|
||||
"description": "Duration in seconds.",
|
||||
},
|
||||
"lyrics": map[string]interface{}{
|
||||
"type": "string",
|
||||
"description": "Lyrics for music generation. Use [Verse], [Chorus] tags.",
|
||||
},
|
||||
"instrumental": map[string]interface{}{
|
||||
"type": "boolean",
|
||||
"description": "No vocals (default: false).",
|
||||
},
|
||||
"provider": map[string]interface{}{
|
||||
"type": "string",
|
||||
"description": "Force a specific provider (e.g. 'minimax').",
|
||||
},
|
||||
},
|
||||
"required": []string{"prompt"},
|
||||
}
|
||||
}
|
||||
|
||||
func (t *CreateAudioTool) Execute(ctx context.Context, args map[string]interface{}) *Result {
|
||||
prompt, _ := args["prompt"].(string)
|
||||
if prompt == "" {
|
||||
return ErrorResult("prompt is required")
|
||||
}
|
||||
|
||||
audioType, _ := args["type"].(string)
|
||||
if audioType == "" {
|
||||
audioType = "music"
|
||||
}
|
||||
|
||||
duration := 0
|
||||
if d, ok := args["duration"].(float64); ok {
|
||||
duration = int(d)
|
||||
}
|
||||
|
||||
lyrics, _ := args["lyrics"].(string)
|
||||
instrumental, _ := args["instrumental"].(bool)
|
||||
forceProvider, _ := args["provider"].(string)
|
||||
|
||||
var audioBytes []byte
|
||||
var usage *providers.Usage
|
||||
var providerName, model string
|
||||
|
||||
// Sound effects: route directly to ElevenLabs if credentials available.
|
||||
if audioType == "sound_effect" {
|
||||
if t.elevenlabsAPIKey == "" {
|
||||
return ErrorResult("sound_effect generation requires ElevenLabs credentials (elevenlabs.api_key in config)")
|
||||
}
|
||||
var err error
|
||||
audioBytes, err = t.callElevenLabsSoundEffect(ctx, prompt, duration)
|
||||
if err != nil {
|
||||
return ErrorResult(fmt.Sprintf("ElevenLabs sound effect generation failed: %v", err))
|
||||
}
|
||||
providerName = "elevenlabs"
|
||||
model = "sound-generation"
|
||||
} else {
|
||||
// Music: use the provider chain.
|
||||
chain := ResolveMediaProviderChain(ctx, "create_audio", forceProvider, "",
|
||||
audioGenProviderPriority, audioGenModelDefaults, t.registry)
|
||||
|
||||
// Inject prompt and music-specific params into each chain entry.
|
||||
for i := range chain {
|
||||
if chain[i].Params == nil {
|
||||
chain[i].Params = make(map[string]any)
|
||||
}
|
||||
chain[i].Params["prompt"] = prompt
|
||||
chain[i].Params["lyrics"] = lyrics
|
||||
chain[i].Params["instrumental"] = instrumental
|
||||
if duration > 0 {
|
||||
chain[i].Params["duration"] = duration
|
||||
}
|
||||
}
|
||||
|
||||
// Override timeout to 300s for music generation (slow API).
|
||||
for i := range chain {
|
||||
if chain[i].Timeout < 300 {
|
||||
chain[i].Timeout = 300
|
||||
}
|
||||
}
|
||||
|
||||
chainResult, err := ExecuteWithChain(ctx, chain, t.registry, t.callProvider)
|
||||
if err != nil {
|
||||
return ErrorResult(fmt.Sprintf("audio generation failed: %v", err))
|
||||
}
|
||||
audioBytes = chainResult.Data
|
||||
usage = chainResult.Usage
|
||||
providerName = chainResult.Provider
|
||||
model = chainResult.Model
|
||||
}
|
||||
|
||||
// Save to workspace under date-based folder.
|
||||
workspace := ToolWorkspaceFromCtx(ctx)
|
||||
if workspace == "" {
|
||||
workspace = os.TempDir()
|
||||
}
|
||||
dateDir := filepath.Join(workspace, "generated", time.Now().Format("2006-01-02"))
|
||||
if err := os.MkdirAll(dateDir, 0755); err != nil {
|
||||
return ErrorResult(fmt.Sprintf("failed to create output directory: %v", err))
|
||||
}
|
||||
audioPath := filepath.Join(dateDir, fmt.Sprintf("goclaw_gen_%d.mp3", time.Now().UnixNano()))
|
||||
if err := os.WriteFile(audioPath, audioBytes, 0644); err != nil {
|
||||
return ErrorResult(fmt.Sprintf("failed to save generated audio: %v", err))
|
||||
}
|
||||
|
||||
slog.Info("create_audio: audio saved", "path", audioPath, "provider", providerName, "type", audioType)
|
||||
|
||||
result := &Result{ForLLM: fmt.Sprintf("MEDIA:%s", audioPath)}
|
||||
result.Media = []bus.MediaFile{{Path: audioPath, MimeType: "audio/mpeg"}}
|
||||
result.Deliverable = fmt.Sprintf("[Generated audio: %s]\nPrompt: %s", filepath.Base(audioPath), prompt)
|
||||
result.Provider = providerName
|
||||
result.Model = model
|
||||
if usage != nil {
|
||||
result.Usage = usage
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// callProvider dispatches to the correct music generation implementation based on provider type.
|
||||
func (t *CreateAudioTool) callProvider(ctx context.Context, cp credentialProvider, providerName, model string, params map[string]any) ([]byte, *providers.Usage, error) {
|
||||
prompt := GetParamString(params, "prompt", "")
|
||||
|
||||
slog.Info("create_audio: calling music generation API",
|
||||
"provider", providerName, "model", model)
|
||||
|
||||
switch ProviderTypeFromName(providerName) {
|
||||
case "minimax":
|
||||
return callMinimaxMusicGen(ctx, cp.APIKey(), cp.APIBase(), model, prompt, params)
|
||||
case "suno":
|
||||
return callSunoMusicGen(ctx, cp.APIKey(), cp.APIBase(), model, prompt, params)
|
||||
default:
|
||||
return nil, nil, fmt.Errorf("unsupported provider for audio generation: %q", providerName)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// callElevenLabsSoundEffect generates a sound effect via the ElevenLabs sound generation API.
|
||||
// Endpoint: POST {baseURL}/v1/sound-generation
|
||||
// Auth: xi-api-key header (NOT Bearer token).
|
||||
// Response: binary audio data directly (not JSON).
|
||||
func (t *CreateAudioTool) callElevenLabsSoundEffect(ctx context.Context, prompt string, durationSeconds int) ([]byte, error) {
|
||||
baseURL := t.elevenlabsBaseURL
|
||||
if baseURL == "" {
|
||||
baseURL = "https://api.elevenlabs.io"
|
||||
}
|
||||
|
||||
// Cap duration at ElevenLabs max of 30 seconds.
|
||||
if durationSeconds > 30 {
|
||||
durationSeconds = 30
|
||||
}
|
||||
|
||||
body := map[string]interface{}{
|
||||
"text": prompt,
|
||||
"output_format": "mp3_44100_128",
|
||||
"prompt_influence": 0.3,
|
||||
}
|
||||
if durationSeconds > 0 {
|
||||
body["duration_seconds"] = durationSeconds
|
||||
}
|
||||
|
||||
jsonBody, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal request: %w", err)
|
||||
}
|
||||
|
||||
url := strings.TrimRight(baseURL, "/") + "/v1/sound-generation"
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonBody))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("xi-api-key", t.elevenlabsAPIKey)
|
||||
|
||||
client := &http.Client{Timeout: 60 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("http request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
errBody, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("ElevenLabs API error %d: %s", resp.StatusCode, truncateBytes(errBody, 500))
|
||||
}
|
||||
|
||||
audioBytes, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read audio response: %w", err)
|
||||
}
|
||||
if len(audioBytes) == 0 {
|
||||
return nil, fmt.Errorf("empty audio response from ElevenLabs")
|
||||
}
|
||||
|
||||
return audioBytes, nil
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/providers"
|
||||
)
|
||||
|
||||
// callMinimaxMusicGen calls the MiniMax music generation API.
|
||||
// Endpoint: POST {apiBase}/music_generation
|
||||
// Response: data.audio or data.music (URL to audio file, downloaded and returned as bytes).
|
||||
func callMinimaxMusicGen(ctx context.Context, apiKey, apiBase, model, prompt string, params map[string]any) ([]byte, *providers.Usage, error) {
|
||||
lyrics := GetParamString(params, "lyrics", "")
|
||||
instrumental := GetParamBool(params, "instrumental", false)
|
||||
lyricsOptimizer := GetParamBool(params, "lyrics_optimizer", false)
|
||||
sampleRate := GetParamString(params, "sample_rate", "44100")
|
||||
bitrate := GetParamString(params, "bitrate", "320k")
|
||||
|
||||
body := map[string]interface{}{
|
||||
"model": model,
|
||||
"prompt": prompt,
|
||||
"is_instrumental": instrumental,
|
||||
"lyrics_optimizer": lyricsOptimizer,
|
||||
"output_format": "url",
|
||||
"audio_setting": map[string]interface{}{
|
||||
"sample_rate": sampleRate,
|
||||
"bitrate": bitrate,
|
||||
"format": "mp3",
|
||||
},
|
||||
}
|
||||
if lyrics != "" {
|
||||
body["lyrics"] = lyrics
|
||||
}
|
||||
|
||||
jsonBody, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("marshal request: %w", err)
|
||||
}
|
||||
|
||||
url := strings.TrimRight(apiBase, "/") + "/music_generation"
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonBody))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+apiKey)
|
||||
|
||||
client := &http.Client{}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("http request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("read response: %w", err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, nil, fmt.Errorf("API error %d: %s", resp.StatusCode, truncateBytes(respBody, 500))
|
||||
}
|
||||
|
||||
var minimaxResp struct {
|
||||
Data *struct {
|
||||
Audio string `json:"audio"`
|
||||
Music string `json:"music"`
|
||||
} `json:"data"`
|
||||
BaseResp *struct {
|
||||
StatusCode int `json:"status_code"`
|
||||
StatusMsg string `json:"status_msg"`
|
||||
} `json:"base_resp"`
|
||||
}
|
||||
if err := json.Unmarshal(respBody, &minimaxResp); err != nil {
|
||||
return nil, nil, fmt.Errorf("parse response: %w", err)
|
||||
}
|
||||
|
||||
if minimaxResp.BaseResp != nil && minimaxResp.BaseResp.StatusCode != 0 {
|
||||
return nil, nil, fmt.Errorf("MiniMax API error %d: %s",
|
||||
minimaxResp.BaseResp.StatusCode, minimaxResp.BaseResp.StatusMsg)
|
||||
}
|
||||
|
||||
if minimaxResp.Data == nil {
|
||||
return nil, nil, fmt.Errorf("no data in MiniMax music response")
|
||||
}
|
||||
|
||||
audioURL := minimaxResp.Data.Audio
|
||||
if audioURL == "" {
|
||||
audioURL = minimaxResp.Data.Music
|
||||
}
|
||||
if audioURL == "" {
|
||||
return nil, nil, fmt.Errorf("no audio URL in MiniMax music response")
|
||||
}
|
||||
|
||||
// Download the audio file from the returned URL.
|
||||
dlReq, err := http.NewRequestWithContext(ctx, "GET", audioURL, nil)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create download request: %w", err)
|
||||
}
|
||||
|
||||
dlClient := &http.Client{}
|
||||
dlResp, err := dlClient.Do(dlReq)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("download audio: %w", err)
|
||||
}
|
||||
defer dlResp.Body.Close()
|
||||
|
||||
if dlResp.StatusCode != http.StatusOK {
|
||||
dlBody, _ := io.ReadAll(dlResp.Body)
|
||||
return nil, nil, fmt.Errorf("download error %d: %s", dlResp.StatusCode, truncateBytes(dlBody, 300))
|
||||
}
|
||||
|
||||
audioBytes, err := limitedReadAll(dlResp.Body, maxMediaDownloadBytes)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("read audio data: %w", err)
|
||||
}
|
||||
|
||||
return audioBytes, nil, nil
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/providers"
|
||||
)
|
||||
|
||||
// callSunoMusicGen generates music via the Suno API.
|
||||
// Suno must be registered as an LLM provider with provider_type "suno".
|
||||
func callSunoMusicGen(ctx context.Context, apiKey, apiBase, model, prompt string, params map[string]any) ([]byte, *providers.Usage, error) {
|
||||
// TODO: Implement Suno music generation
|
||||
return nil, nil, fmt.Errorf("suno music generation not yet implemented")
|
||||
}
|
||||
Reference in new issue
Block a user