From f4cc595e50a78610f57332935d10f7f398703812 Mon Sep 17 00:00:00 2001 From: viettranx Date: Tue, 14 Apr 2026 22:31:42 +0700 Subject: [PATCH] feat(audio): add unified audio manager with 4 provider interfaces Introduce internal/audio package with Manager orchestrating TTS, STT, Music, and SFX providers via 4 interfaces. Phase 1 wires TTS providers (ElevenLabs, OpenAI, Edge, MiniMax) and ElevenLabs SFX; STT/Music wiring deferred to later phases. ElevenLabs TTS and SFX share an xi-api-key HTTP client. --- internal/audio/edge/tts.go | 84 ++++++++++++++ internal/audio/elevenlabs/client.go | 70 ++++++++++++ internal/audio/elevenlabs/config.go | 31 +++++ internal/audio/elevenlabs/sfx.go | 72 ++++++++++++ internal/audio/elevenlabs/tts.go | 74 ++++++++++++ internal/audio/manager.go | 149 ++++++++++++++++++++++++ internal/audio/manager_auto.go | 70 ++++++++++++ internal/audio/manager_smoke_test.go | 46 ++++++++ internal/audio/minimax/tts.go | 163 +++++++++++++++++++++++++++ internal/audio/openai/tts.go | 124 ++++++++++++++++++++ internal/audio/text_processing.go | 26 +++++ internal/audio/types.go | 112 ++++++++++++++++++ 12 files changed, 1021 insertions(+) create mode 100644 internal/audio/edge/tts.go create mode 100644 internal/audio/elevenlabs/client.go create mode 100644 internal/audio/elevenlabs/config.go create mode 100644 internal/audio/elevenlabs/sfx.go create mode 100644 internal/audio/elevenlabs/tts.go create mode 100644 internal/audio/manager.go create mode 100644 internal/audio/manager_auto.go create mode 100644 internal/audio/manager_smoke_test.go create mode 100644 internal/audio/minimax/tts.go create mode 100644 internal/audio/openai/tts.go create mode 100644 internal/audio/text_processing.go create mode 100644 internal/audio/types.go diff --git a/internal/audio/edge/tts.go b/internal/audio/edge/tts.go new file mode 100644 index 00000000..a798496d --- /dev/null +++ b/internal/audio/edge/tts.go @@ -0,0 +1,84 @@ +// Package edge implements TTS via the Microsoft Edge TTS CLI (free, no API key). +// Requires the `edge-tts` Python CLI: `pip install edge-tts`. +package edge + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "time" + + "github.com/nextlevelbuilder/goclaw/internal/audio" +) + +// Config configures the Edge TTS provider. +type Config struct { + Voice string // default "en-US-MichelleNeural" + Rate string // speech rate, e.g. "+0%" + TimeoutMs int +} + +// Provider implements audio.TTSProvider via the edge-tts CLI. +type Provider struct { + voice string + rate string + timeoutMs int +} + +// NewProvider returns an Edge TTS provider with defaults applied. +func NewProvider(cfg Config) *Provider { + p := &Provider{ + voice: cfg.Voice, + rate: cfg.Rate, + timeoutMs: cfg.TimeoutMs, + } + if p.voice == "" { + p.voice = "en-US-MichelleNeural" + } + if p.timeoutMs <= 0 { + p.timeoutMs = 30000 + } + return p +} + +// Name returns the stable provider identifier used by the Manager. +func (p *Provider) Name() string { return "edge" } + +// Synthesize shells out to edge-tts. Output is always MP3 +// (edge-tts default format: audio-24khz-48kbitrate-mono-mp3). +func (p *Provider) Synthesize(ctx context.Context, text string, _ audio.TTSOptions) (*audio.SynthResult, error) { + tmpDir := os.TempDir() + outPath := filepath.Join(tmpDir, fmt.Sprintf("tts-%d.mp3", time.Now().UnixNano())) + defer os.Remove(outPath) + + args := []string{ + "--voice", p.voice, + "--text", text, + "--write-media", outPath, + } + if p.rate != "" { + args = append(args, "--rate", p.rate) + } + + timeout := time.Duration(p.timeoutMs) * time.Millisecond + cmdCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + cmd := exec.CommandContext(cmdCtx, "edge-tts", args...) + if output, err := cmd.CombinedOutput(); err != nil { + return nil, fmt.Errorf("edge-tts failed: %w (output: %s)", err, string(output)) + } + + audioBytes, err := os.ReadFile(outPath) + if err != nil { + return nil, fmt.Errorf("read edge-tts output: %w", err) + } + + return &audio.SynthResult{ + Audio: audioBytes, + Extension: "mp3", + MimeType: "audio/mpeg", + }, nil +} diff --git a/internal/audio/elevenlabs/client.go b/internal/audio/elevenlabs/client.go new file mode 100644 index 00000000..08453fef --- /dev/null +++ b/internal/audio/elevenlabs/client.go @@ -0,0 +1,70 @@ +package elevenlabs + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// client is the shared HTTP client used by the TTS and SFX providers. It +// encapsulates the ElevenLabs `xi-api-key` auth scheme + common error handling. +type client struct { + apiKey string + baseURL string + timeoutMs int +} + +func newClient(apiKey, baseURL string, timeoutMs int) *client { + if baseURL == "" { + baseURL = "https://api.elevenlabs.io" + } + if timeoutMs <= 0 { + timeoutMs = 30000 + } + return &client{apiKey: apiKey, baseURL: baseURL, timeoutMs: timeoutMs} +} + +// postJSON performs a POST with JSON body to {baseURL}/path and returns the +// raw response bytes on 200 OK. Non-200 responses surface as errors with the +// upstream body appended — matches legacy behavior for debuggability. +func (c *client) postJSON(ctx context.Context, path string, body []byte, customTimeout time.Duration) ([]byte, error) { + url := strings.TrimRight(c.baseURL, "/") + path + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("xi-api-key", c.apiKey) + + timeout := customTimeout + if timeout <= 0 { + timeout = time.Duration(c.timeoutMs) * time.Millisecond + } + hc := &http.Client{Timeout: timeout} + resp, err := hc.Do(req) + if err != nil { + return nil, fmt.Errorf("http request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + errBody, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("ElevenLabs API error %d: %s", resp.StatusCode, truncate(errBody, 500)) + } + out, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("read response: %w", err) + } + return out, nil +} + +func truncate(b []byte, n int) string { + if len(b) <= n { + return string(b) + } + return string(b[:n]) + "...(truncated)" +} diff --git a/internal/audio/elevenlabs/config.go b/internal/audio/elevenlabs/config.go new file mode 100644 index 00000000..132e002f --- /dev/null +++ b/internal/audio/elevenlabs/config.go @@ -0,0 +1,31 @@ +// Package elevenlabs implements the ElevenLabs audio API for TTS and SFX. +// TTS ships in Phase 1; SFX migrated from internal/tools/create_audio_elevenlabs.go. +// Music generation via /v1/music/compose lands in Phase 3. +package elevenlabs + +// Config bundles credentials + TTS defaults for ElevenLabs. Shared by the +// TTS and SFX providers — SFX ignores Voice/Model fields. +type Config struct { + APIKey string + BaseURL string // default "https://api.elevenlabs.io" + VoiceID string // default "pMsXgVXv3BLzUgSXRplE" (TTS only) + ModelID string // default "eleven_multilingual_v2" (TTS only) + TimeoutMs int // default 30000 +} + +// defaults fills in blank fields with the values used by the legacy tts package. +func (c Config) withDefaults() Config { + if c.BaseURL == "" { + c.BaseURL = "https://api.elevenlabs.io" + } + if c.VoiceID == "" { + c.VoiceID = "pMsXgVXv3BLzUgSXRplE" + } + if c.ModelID == "" { + c.ModelID = "eleven_multilingual_v2" + } + if c.TimeoutMs <= 0 { + c.TimeoutMs = 30000 + } + return c +} diff --git a/internal/audio/elevenlabs/sfx.go b/internal/audio/elevenlabs/sfx.go new file mode 100644 index 00000000..b20c5619 --- /dev/null +++ b/internal/audio/elevenlabs/sfx.go @@ -0,0 +1,72 @@ +package elevenlabs + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/nextlevelbuilder/goclaw/internal/audio" +) + +// sfxMaxDurationSeconds is ElevenLabs' upper limit for /v1/sound-generation. +const sfxMaxDurationSeconds = 30 + +// SFXProvider generates short sound effects via POST /v1/sound-generation. +// Migrated from internal/tools/create_audio_elevenlabs.go — byte-for-byte +// identical request payload + 60s timeout. +type SFXProvider struct { + cfg Config + c *client +} + +// NewSFXProvider returns an ElevenLabs SFX provider. +func NewSFXProvider(cfg Config) *SFXProvider { + cfg = cfg.withDefaults() + return &SFXProvider{ + cfg: cfg, + c: newClient(cfg.APIKey, cfg.BaseURL, cfg.TimeoutMs), + } +} + +// Name returns the stable provider identifier used by the Manager. +func (p *SFXProvider) Name() string { return "elevenlabs" } + +// GenerateSFX produces an MP3 sound effect from opts.Prompt. Duration is +// capped at sfxMaxDurationSeconds (ElevenLabs limit). +func (p *SFXProvider) GenerateSFX(ctx context.Context, opts audio.SFXOptions) (*audio.AudioResult, error) { + duration := opts.Duration + if duration > sfxMaxDurationSeconds { + duration = sfxMaxDurationSeconds + } + + body := map[string]any{ + "text": opts.Prompt, + "output_format": "mp3_44100_128", + "prompt_influence": 0.3, + } + if duration > 0 { + body["duration_seconds"] = duration + } + + jsonBody, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("marshal sfx request: %w", err) + } + + // Sound-generation is bursty — legacy code used a fixed 60s timeout, not + // the per-call TimeoutMs. Preserve that exactly. + audioBytes, err := p.c.postJSON(ctx, "/v1/sound-generation", jsonBody, 60*time.Second) + if err != nil { + return nil, fmt.Errorf("elevenlabs sfx: %w", err) + } + if len(audioBytes) == 0 { + return nil, fmt.Errorf("empty audio response from ElevenLabs") + } + + return &audio.AudioResult{ + Audio: audioBytes, + Extension: "mp3", + MimeType: "audio/mpeg", + }, nil +} diff --git a/internal/audio/elevenlabs/tts.go b/internal/audio/elevenlabs/tts.go new file mode 100644 index 00000000..18dc50c9 --- /dev/null +++ b/internal/audio/elevenlabs/tts.go @@ -0,0 +1,74 @@ +package elevenlabs + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/nextlevelbuilder/goclaw/internal/audio" +) + +// TTSProvider synthesizes text via POST /v1/text-to-speech/{voice_id}. +// Legacy name: internal/tts.ElevenLabsProvider. +type TTSProvider struct { + cfg Config + c *client +} + +// NewTTSProvider returns an ElevenLabs TTS provider backed by the shared client. +func NewTTSProvider(cfg Config) *TTSProvider { + cfg = cfg.withDefaults() + return &TTSProvider{ + cfg: cfg, + c: newClient(cfg.APIKey, cfg.BaseURL, cfg.TimeoutMs), + } +} + +// Name returns the stable provider identifier used by the Manager. +func (p *TTSProvider) Name() string { return "elevenlabs" } + +// Synthesize converts text to audio. Opts.Voice/Opts.Model override the +// configured defaults; Opts.Format="opus" switches to Ogg Opus output. +func (p *TTSProvider) Synthesize(ctx context.Context, text string, opts audio.TTSOptions) (*audio.SynthResult, error) { + voiceID := opts.Voice + if voiceID == "" { + voiceID = p.cfg.VoiceID + } + modelID := opts.Model + if modelID == "" { + modelID = p.cfg.ModelID + } + + // Output format: MP3 128kbps default, Opus 64kbps for Telegram voice. + outputFormat, ext, mime := "mp3_44100_128", "mp3", "audio/mpeg" + if opts.Format == "opus" { + outputFormat, ext, mime = "opus_48000_64", "ogg", "audio/ogg" + } + + body := map[string]any{ + "text": text, + "model_id": modelID, + "voice_settings": map[string]any{ + "stability": 0.5, + "similarity_boost": 0.75, + "style": 0.0, + "use_speaker_boost": true, + }, + } + bodyJSON, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("marshal elevenlabs tts request: %w", err) + } + + path := fmt.Sprintf("/v1/text-to-speech/%s?output_format=%s", voiceID, outputFormat) + audioBytes, err := p.c.postJSON(ctx, path, bodyJSON, 0) + if err != nil { + return nil, fmt.Errorf("elevenlabs tts: %w", err) + } + + return &audio.SynthResult{ + Audio: audioBytes, + Extension: ext, + MimeType: mime, + }, nil +} diff --git a/internal/audio/manager.go b/internal/audio/manager.go new file mode 100644 index 00000000..b863ae1b --- /dev/null +++ b/internal/audio/manager.go @@ -0,0 +1,149 @@ +package audio + +import ( + "context" + "fmt" + "log/slog" +) + +// Manager orchestrates audio providers across TTS, STT, Music, and SFX +// operations. Each op has its own provider map + primary/fallback chain. +// +// Phase 1 exercises the TTS path end-to-end. STT/Music/SFX maps and chains +// are present but empty (providers register in Phase 3/4). +type Manager struct { + ttsProviders map[string]TTSProvider + sttProviders map[string]STTProvider + musicProviders map[string]MusicProvider + sfxProviders map[string]SFXProvider + + primary string // primary TTS provider + sttChain []string // STT fallback order (Phase 4) + musicChain []string // Music fallback order (Phase 3) + + auto AutoMode + mode Mode + maxLength int // max text length before truncation (default 1500) + timeoutMs int // provider timeout (default 30000) +} + +// ManagerConfig configures the audio manager. Preserved from legacy TTS +// package — new STT/Music fields are set via RegisterSTT/RegisterMusic and +// (optionally) cfg.Audio in config_audio.go. +type ManagerConfig struct { + Primary string // primary TTS provider name + Auto AutoMode // auto-apply mode (default "off") + Mode Mode // "final" or "all" (default "final") + MaxLength int // default 1500 + TimeoutMs int // default 30000 +} + +// NewManager creates an audio manager with empty provider maps. +func NewManager(cfg ManagerConfig) *Manager { + m := &Manager{ + ttsProviders: make(map[string]TTSProvider), + sttProviders: make(map[string]STTProvider), + musicProviders: make(map[string]MusicProvider), + sfxProviders: make(map[string]SFXProvider), + primary: cfg.Primary, + auto: cfg.Auto, + mode: cfg.Mode, + maxLength: cfg.MaxLength, + timeoutMs: cfg.TimeoutMs, + } + if m.auto == "" { + m.auto = AutoOff + } + if m.mode == "" { + m.mode = ModeFinal + } + if m.maxLength <= 0 { + m.maxLength = 1500 + } + if m.timeoutMs <= 0 { + m.timeoutMs = 30000 + } + return m +} + +// ---- Registration ---- + +// RegisterTTS adds a TTS provider. If no primary is set, the first registered +// provider becomes primary — matches legacy tts.Manager.RegisterProvider. +func (m *Manager) RegisterTTS(p TTSProvider) { + m.ttsProviders[p.Name()] = p + if m.primary == "" { + m.primary = p.Name() + } +} + +// RegisterProvider is a backward-compat alias for RegisterTTS — lets pre-Phase-1 +// callers that go through tts.Manager (= audio.Manager via alias) keep working. +func (m *Manager) RegisterProvider(p TTSProvider) { m.RegisterTTS(p) } + +// RegisterSTT adds an STT provider (Phase 4). +func (m *Manager) RegisterSTT(p STTProvider) { + m.sttProviders[p.Name()] = p +} + +// RegisterMusic adds a music provider (Phase 3). +func (m *Manager) RegisterMusic(p MusicProvider) { + m.musicProviders[p.Name()] = p +} + +// RegisterSFX adds an SFX provider (Phase 3). +func (m *Manager) RegisterSFX(p SFXProvider) { + m.sfxProviders[p.Name()] = p +} + +// ---- Introspection ---- + +// GetProvider returns a TTS provider by name. Preserved from legacy API. +func (m *Manager) GetProvider(name string) (TTSProvider, bool) { + p, ok := m.ttsProviders[name] + return p, ok +} + +// PrimaryProvider returns the primary TTS provider name. +func (m *Manager) PrimaryProvider() string { return m.primary } + +// AutoMode returns the current auto-apply mode. +func (m *Manager) AutoMode() AutoMode { return m.auto } + +// HasProviders reports whether any TTS provider is registered. +func (m *Manager) HasProviders() bool { return len(m.ttsProviders) > 0 } + +// ---- TTS dispatch ---- + +// Synthesize uses the primary provider. +func (m *Manager) Synthesize(ctx context.Context, text string, opts TTSOptions) (*SynthResult, error) { + p, ok := m.ttsProviders[m.primary] + if !ok { + return nil, fmt.Errorf("tts provider not found: %s", m.primary) + } + return p.Synthesize(ctx, text, opts) +} + +// SynthesizeWithFallback tries primary first, then any other registered +// provider on error. Returns first success or aggregate failure. +func (m *Manager) SynthesizeWithFallback(ctx context.Context, text string, opts TTSOptions) (*SynthResult, error) { + if p, ok := m.ttsProviders[m.primary]; ok { + if result, err := p.Synthesize(ctx, text, opts); err == nil { + return result, nil + } else { + slog.Warn("tts primary provider failed, trying fallback", "provider", m.primary, "error", err) + } + } + for name, p := range m.ttsProviders { + if name == m.primary { + continue + } + result, err := p.Synthesize(ctx, text, opts) + if err == nil { + slog.Info("tts fallback succeeded", "provider", name) + return result, nil + } + slog.Warn("tts fallback provider failed", "provider", name, "error", err) + } + return nil, fmt.Errorf("all tts providers failed") +} diff --git a/internal/audio/manager_auto.go b/internal/audio/manager_auto.go new file mode 100644 index 00000000..19d6ed7b --- /dev/null +++ b/internal/audio/manager_auto.go @@ -0,0 +1,70 @@ +package audio + +import ( + "context" + "log/slog" + "strings" +) + +// MaybeApply inspects auto-mode and conditionally applies TTS to a reply. +// Returns (result, true) on success, (nil, false) when auto is disabled, the +// reply type is filtered out, content fails validation, or synthesis fails. +// +// Parameters: +// - text: the reply text to potentially convert +// - channel: origin channel ("telegram" switches format to opus) +// - isVoiceInbound: whether the user's inbound message was voice +// - kind: "tool", "block", or "final" +func (m *Manager) MaybeApply(ctx context.Context, text, channel string, isVoiceInbound bool, kind string) (*SynthResult, bool) { + if m.auto == AutoOff { + return nil, false + } + + // Mode filter: ModeFinal skips tool/block replies. + if m.mode == ModeFinal && (kind == "tool" || kind == "block") { + return nil, false + } + + switch m.auto { + case AutoInbound: + if !isVoiceInbound { + return nil, false + } + case AutoTagged: + if !strings.Contains(text, "[[tts]]") && !strings.Contains(text, "[[tts:") { + return nil, false + } + case AutoAlways: + // Always apply. + default: + return nil, false + } + + // Content validation (matches legacy TTS behavior). + cleanText := stripMarkdown(text) + cleanText = stripTtsDirectives(cleanText) + cleanText = strings.TrimSpace(cleanText) + + if len(cleanText) < 10 { + return nil, false + } + if strings.Contains(cleanText, "MEDIA:") { + return nil, false + } + + if len(cleanText) > m.maxLength { + cleanText = cleanText[:m.maxLength] + "..." + } + + opts := TTSOptions{} + if channel == "telegram" { + opts.Format = "opus" // Telegram voice bubbles need opus + } + + result, err := m.SynthesizeWithFallback(ctx, cleanText, opts) + if err != nil { + slog.Warn("tts auto-apply failed", "error", err) + return nil, false + } + return result, true +} diff --git a/internal/audio/manager_smoke_test.go b/internal/audio/manager_smoke_test.go new file mode 100644 index 00000000..ecc77038 --- /dev/null +++ b/internal/audio/manager_smoke_test.go @@ -0,0 +1,46 @@ +package audio_test + +// Manager skeleton smoke test. Red-first: this file fails to compile until +// internal/audio is created with Manager, TTSProvider, and the 4 Register* +// functions. Once compile passes, tests exercise the minimum surface needed +// by phase-01 callers (NewManager, RegisterTTS, HasProviders, zero-value +// chains). + +import ( + "testing" + + "github.com/nextlevelbuilder/goclaw/internal/audio" +) + +func TestNewManager_ReturnsNonNil(t *testing.T) { + mgr := audio.NewManager(audio.ManagerConfig{Primary: "test"}) + if mgr == nil { + t.Fatal("NewManager returned nil") + } +} + +func TestManager_ZeroValueChains_AreEmpty(t *testing.T) { + mgr := audio.NewManager(audio.ManagerConfig{}) + if mgr.HasProviders() { + t.Fatal("fresh Manager should not report providers") + } + if _, ok := mgr.GetProvider("anything"); ok { + t.Fatal("fresh Manager should not resolve providers") + } +} + +type stubTTS struct{ name string } + +func (s stubTTS) Name() string { return s.name } +func (s stubTTS) Synthesize(_ any, _ string, _ audio.TTSOptions) (*audio.SynthResult, error) { + return &audio.SynthResult{Extension: "mp3", MimeType: "audio/mpeg"}, nil +} + +// NOTE: stubTTS.Synthesize takes `any` for ctx to avoid importing context in +// the smoke test — the TTSProvider interface will enforce context.Context via +// compile check when we add `var _ audio.TTSProvider = stubTTS{}`. +// +// Smoke test intentionally omits the `var _ audio.TTSProvider = stubTTS{}` +// assertion because stubTTS uses `any` for ctx. Provider-interface conformance +// is covered by the alias_test (tts.Provider alias of audio.TTSProvider) and +// by production provider packages (elevenlabs, openai, edge, minimax). diff --git a/internal/audio/minimax/tts.go b/internal/audio/minimax/tts.go new file mode 100644 index 00000000..bb448e08 --- /dev/null +++ b/internal/audio/minimax/tts.go @@ -0,0 +1,163 @@ +// Package minimax implements TTS via the MiniMax T2A v2 API. +// Docs: https://platform.minimax.io/docs/api-reference/speech-t2a-intro +package minimax + +import ( + "bytes" + "context" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/nextlevelbuilder/goclaw/internal/audio" +) + +// Config configures the MiniMax TTS provider. +type Config struct { + APIKey string + GroupID string // required query param + APIBase string // default "https://api.minimax.io/v1" + Model string // default "speech-02-hd" + VoiceID string // default "Wise_Woman" + TimeoutMs int +} + +// Provider implements audio.TTSProvider via MiniMax /t2a_v2. +type Provider struct { + apiKey string + groupID string + apiBase string + model string + voiceID string + timeoutMs int +} + +// NewProvider returns a MiniMax TTS provider with defaults applied. +func NewProvider(cfg Config) *Provider { + p := &Provider{ + apiKey: cfg.APIKey, + groupID: cfg.GroupID, + apiBase: cfg.APIBase, + model: cfg.Model, + voiceID: cfg.VoiceID, + timeoutMs: cfg.TimeoutMs, + } + if p.apiBase == "" { + p.apiBase = "https://api.minimax.io/v1" + } + if p.model == "" { + p.model = "speech-02-hd" + } + if p.voiceID == "" { + p.voiceID = "Wise_Woman" + } + if p.timeoutMs <= 0 { + p.timeoutMs = 30000 + } + return p +} + +// Name returns the stable provider identifier used by the Manager. +func (p *Provider) Name() string { return "minimax" } + +// Synthesize calls MiniMax t2a_v2 (non-streaming). MiniMax returns hex-encoded +// audio in response.data.audio — we decode before returning. +func (p *Provider) Synthesize(ctx context.Context, text string, opts audio.TTSOptions) (*audio.SynthResult, error) { + voiceID := opts.Voice + if voiceID == "" { + voiceID = p.voiceID + } + model := opts.Model + if model == "" { + model = p.model + } + + audioFormat, ext, mime := "mp3", "mp3", "audio/mpeg" + if opts.Format == "opus" || opts.Format == "pcm" || opts.Format == "flac" || opts.Format == "wav" { + audioFormat = opts.Format + switch opts.Format { + case "pcm": + ext, mime = "pcm", "audio/pcm" + case "flac": + ext, mime = "flac", "audio/flac" + case "wav": + ext, mime = "wav", "audio/wav" + } + } + + body := map[string]any{ + "text": text, + "model": model, + "stream": false, + "voice_setting": map[string]any{ + "voice_id": voiceID, + "speed": 1.0, + "pitch": 0, + }, + "audio_setting": map[string]any{ + "format": audioFormat, + }, + } + bodyJSON, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("marshal minimax tts request: %w", err) + } + + url := fmt.Sprintf("%s/t2a_v2?GroupId=%s", p.apiBase, p.groupID) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(bodyJSON)) + if err != nil { + return nil, fmt.Errorf("create minimax tts request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+p.apiKey) + + hc := &http.Client{Timeout: time.Duration(p.timeoutMs) * time.Millisecond} + resp, err := hc.Do(req) + if err != nil { + return nil, fmt.Errorf("minimax tts request failed: %w", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("read minimax tts response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("minimax tts error %d: %s", resp.StatusCode, string(respBody)) + } + + var apiResp miniMaxResponse + if err := json.Unmarshal(respBody, &apiResp); err != nil { + return nil, fmt.Errorf("parse minimax tts response: %w", err) + } + if apiResp.BaseResp.StatusCode != 0 { + return nil, fmt.Errorf("minimax tts api error %d: %s", apiResp.BaseResp.StatusCode, apiResp.BaseResp.StatusMsg) + } + if apiResp.Data.Audio == "" { + return nil, fmt.Errorf("minimax tts returned empty audio") + } + + audioBytes, err := hex.DecodeString(apiResp.Data.Audio) + if err != nil { + return nil, fmt.Errorf("decode minimax tts audio hex: %w", err) + } + + return &audio.SynthResult{ + Audio: audioBytes, + Extension: ext, + MimeType: mime, + }, nil +} + +type miniMaxResponse struct { + BaseResp struct { + StatusCode int `json:"status_code"` + StatusMsg string `json:"status_msg"` + } `json:"base_resp"` + Data struct { + Audio string `json:"audio"` // hex-encoded audio bytes + } `json:"data"` +} diff --git a/internal/audio/openai/tts.go b/internal/audio/openai/tts.go new file mode 100644 index 00000000..5536b2ac --- /dev/null +++ b/internal/audio/openai/tts.go @@ -0,0 +1,124 @@ +// Package openai implements the OpenAI audio/speech API for TTS. +package openai + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/nextlevelbuilder/goclaw/internal/audio" +) + +// Config bundles credentials + TTS defaults for OpenAI. +type Config struct { + APIKey string + APIBase string // default "https://api.openai.com/v1" + Model string // default "gpt-4o-mini-tts" + Voice string // default "alloy" + TimeoutMs int // default 30000 +} + +// Provider implements audio.TTSProvider for OpenAI. +type Provider struct { + apiKey string + apiBase string + model string + voice string + timeoutMs int +} + +// NewProvider constructs an OpenAI TTS provider with defaults applied. +func NewProvider(cfg Config) *Provider { + p := &Provider{ + apiKey: cfg.APIKey, + apiBase: cfg.APIBase, + model: cfg.Model, + voice: cfg.Voice, + timeoutMs: cfg.TimeoutMs, + } + if p.apiBase == "" { + p.apiBase = "https://api.openai.com/v1" + } + if p.model == "" { + p.model = "gpt-4o-mini-tts" + } + if p.voice == "" { + p.voice = "alloy" + } + if p.timeoutMs <= 0 { + p.timeoutMs = 30000 + } + return p +} + +// Name returns the stable provider identifier used by the Manager. +func (p *Provider) Name() string { return "openai" } + +// Synthesize calls POST {apiBase}/audio/speech. +func (p *Provider) Synthesize(ctx context.Context, text string, opts audio.TTSOptions) (*audio.SynthResult, error) { + voice := opts.Voice + if voice == "" { + voice = p.voice + } + model := opts.Model + if model == "" { + model = p.model + } + format := opts.Format + if format == "" { + format = "mp3" + } + + body := map[string]any{ + "model": model, + "input": text, + "voice": voice, + "response_format": format, + } + bodyJSON, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("marshal openai tts request: %w", err) + } + + url := p.apiBase + "/audio/speech" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(bodyJSON)) + if err != nil { + return nil, fmt.Errorf("create openai tts request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+p.apiKey) + + hc := &http.Client{Timeout: time.Duration(p.timeoutMs) * time.Millisecond} + resp, err := hc.Do(req) + if err != nil { + return nil, fmt.Errorf("openai tts request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + errBody, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("openai tts error %d: %s", resp.StatusCode, string(errBody)) + } + audioBytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("read openai tts response: %w", err) + } + + ext, mime := format, "audio/mpeg" + switch format { + case "opus": + ext, mime = "ogg", "audio/ogg" + case "mp3": + mime = "audio/mpeg" + } + + return &audio.SynthResult{ + Audio: audioBytes, + Extension: ext, + MimeType: mime, + }, nil +} diff --git a/internal/audio/text_processing.go b/internal/audio/text_processing.go new file mode 100644 index 00000000..6c214628 --- /dev/null +++ b/internal/audio/text_processing.go @@ -0,0 +1,26 @@ +package audio + +import "regexp" + +// stripMarkdown removes common markdown formatting so TTS reads prose, not +// syntax characters. Preserves inner text of bold/italic/inline code/links. +func stripMarkdown(text string) string { + text = regexp.MustCompile("(?s)```[^`]*```").ReplaceAllString(text, "") + text = regexp.MustCompile("`([^`]+)`").ReplaceAllString(text, "$1") + text = regexp.MustCompile(`\*\*([^*]+)\*\*`).ReplaceAllString(text, "$1") + text = regexp.MustCompile(`\*([^*]+)\*`).ReplaceAllString(text, "$1") + text = regexp.MustCompile(`__([^_]+)__`).ReplaceAllString(text, "$1") + text = regexp.MustCompile(`_([^_]+)_`).ReplaceAllString(text, "$1") + text = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`).ReplaceAllString(text, "$1") + text = regexp.MustCompile(`(?m)^#+\s+`).ReplaceAllString(text, "") + return text +} + +// stripTtsDirectives removes [[tts...]] markup from text. +// `[[tts:text]]...[[/tts:text]]` blocks keep their inner content. +// Bare `[[tts]]` and `[[tts:something]]` tags are removed entirely. +func stripTtsDirectives(text string) string { + text = regexp.MustCompile(`(?s)\[\[tts:text\]\](.*?)\[\[/tts:text\]\]`).ReplaceAllString(text, "$1") + text = regexp.MustCompile(`\[\[tts(?::[^\]]*)?\]\]`).ReplaceAllString(text, "") + return text +} diff --git a/internal/audio/types.go b/internal/audio/types.go new file mode 100644 index 00000000..0df61550 --- /dev/null +++ b/internal/audio/types.go @@ -0,0 +1,112 @@ +// Package audio unifies TTS, STT, Music, and SFX generation under a single +// Manager. Replaces internal/tts (surface preserved via internal/tts/alias.go +// backward-compat layer). +// +// Phase 1 delivers TTS parity; STT/Music/SFX interfaces ship as stubs — +// implementations land in Phase 3 (Music/SFX) and Phase 4 (STT). +package audio + +import "context" + +// ---- TTS (implemented Phase 1) ---- + +// TTSProvider synthesizes text into audio bytes. +type TTSProvider interface { + Name() string + Synthesize(ctx context.Context, text string, opts TTSOptions) (*SynthResult, error) +} + +// TTSOptions controls TTS synthesis parameters. +type TTSOptions struct { + Voice string // provider-specific voice ID + Model string // provider-specific model ID + Format string // output format: "mp3", "opus", etc. +} + +// SynthResult is the output of a TTS synthesis call. +type SynthResult struct { + Audio []byte // raw audio bytes + Extension string // file extension without dot: "mp3", "opus", "ogg" + MimeType string // e.g. "audio/mpeg", "audio/ogg" +} + +// AutoMode controls when TTS is automatically applied to replies. +type AutoMode string + +const ( + AutoOff AutoMode = "off" // Disabled + AutoAlways AutoMode = "always" // Apply to all eligible replies + AutoInbound AutoMode = "inbound" // Only if user sent audio/voice + AutoTagged AutoMode = "tagged" // Only if reply contains [[tts]] directive +) + +// Mode controls which reply kinds get TTS. +type Mode string + +const ( + ModeFinal Mode = "final" // Only final replies (default) + ModeAll Mode = "all" // All replies including tool/block +) + +// ---- STT (stubs — implementations land in Phase 4) ---- + +// STTProvider transcribes audio bytes to text. +type STTProvider interface { + Name() string + Transcribe(ctx context.Context, in STTInput, opts STTOptions) (*TranscriptResult, error) +} + +// STTInput is the audio to transcribe. At most one of Audio/FilePath is set. +type STTInput struct { + Audio []byte // raw audio bytes (in-memory) + FilePath string // path on disk (used by proxy_stt) + MimeType string // e.g. "audio/ogg" +} + +// STTOptions tunes transcription. +type STTOptions struct { + Language string // BCP-47 hint, empty = auto-detect + Model string // provider-specific model ID +} + +// TranscriptResult is the output of transcription. +type TranscriptResult struct { + Text string + Language string // detected or hinted language +} + +// ---- Music (stubs — implementations land in Phase 3) ---- + +// MusicProvider generates music from prompt + optional lyrics. +type MusicProvider interface { + Name() string + GenerateMusic(ctx context.Context, opts MusicOptions) (*AudioResult, error) +} + +// MusicOptions controls music generation. +type MusicOptions struct { + Prompt string + Lyrics string + Duration int // seconds +} + +// ---- SFX (stubs — implementations land in Phase 3) ---- + +// SFXProvider generates short sound effects from a prompt. +type SFXProvider interface { + Name() string + GenerateSFX(ctx context.Context, opts SFXOptions) (*AudioResult, error) +} + +// SFXOptions controls SFX generation. +type SFXOptions struct { + Prompt string + Duration int // seconds (provider may cap) +} + +// AudioResult is the shared output of music/SFX generation. +type AudioResult struct { + Audio []byte + Extension string + MimeType string +}