mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-11 03:13:24 +00:00
feat(audio): add unified audio manager with 4 provider interfaces
Introduce internal/audio package with Manager orchestrating TTS, STT, Music, and SFX providers via 4 interfaces. Phase 1 wires TTS providers (ElevenLabs, OpenAI, Edge, MiniMax) and ElevenLabs SFX; STT/Music wiring deferred to later phases. ElevenLabs TTS and SFX share an xi-api-key HTTP client.
This commit is contained in:
viettranx committed
1 parent
e79a8bbd39
commit
f4cc595e50
12 files changed
+1021
No files matched your search
@@ -0,0 +1,84 @@
|
||||
// Package edge implements TTS via the Microsoft Edge TTS CLI (free, no API key).
|
||||
// Requires the `edge-tts` Python CLI: `pip install edge-tts`.
|
||||
package edge
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/audio"
|
||||
)
|
||||
|
||||
// Config configures the Edge TTS provider.
|
||||
type Config struct {
|
||||
Voice string // default "en-US-MichelleNeural"
|
||||
Rate string // speech rate, e.g. "+0%"
|
||||
TimeoutMs int
|
||||
}
|
||||
|
||||
// Provider implements audio.TTSProvider via the edge-tts CLI.
|
||||
type Provider struct {
|
||||
voice string
|
||||
rate string
|
||||
timeoutMs int
|
||||
}
|
||||
|
||||
// NewProvider returns an Edge TTS provider with defaults applied.
|
||||
func NewProvider(cfg Config) *Provider {
|
||||
p := &Provider{
|
||||
voice: cfg.Voice,
|
||||
rate: cfg.Rate,
|
||||
timeoutMs: cfg.TimeoutMs,
|
||||
}
|
||||
if p.voice == "" {
|
||||
p.voice = "en-US-MichelleNeural"
|
||||
}
|
||||
if p.timeoutMs <= 0 {
|
||||
p.timeoutMs = 30000
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// Name returns the stable provider identifier used by the Manager.
|
||||
func (p *Provider) Name() string { return "edge" }
|
||||
|
||||
// Synthesize shells out to edge-tts. Output is always MP3
|
||||
// (edge-tts default format: audio-24khz-48kbitrate-mono-mp3).
|
||||
func (p *Provider) Synthesize(ctx context.Context, text string, _ audio.TTSOptions) (*audio.SynthResult, error) {
|
||||
tmpDir := os.TempDir()
|
||||
outPath := filepath.Join(tmpDir, fmt.Sprintf("tts-%d.mp3", time.Now().UnixNano()))
|
||||
defer os.Remove(outPath)
|
||||
|
||||
args := []string{
|
||||
"--voice", p.voice,
|
||||
"--text", text,
|
||||
"--write-media", outPath,
|
||||
}
|
||||
if p.rate != "" {
|
||||
args = append(args, "--rate", p.rate)
|
||||
}
|
||||
|
||||
timeout := time.Duration(p.timeoutMs) * time.Millisecond
|
||||
cmdCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
cmd := exec.CommandContext(cmdCtx, "edge-tts", args...)
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
return nil, fmt.Errorf("edge-tts failed: %w (output: %s)", err, string(output))
|
||||
}
|
||||
|
||||
audioBytes, err := os.ReadFile(outPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read edge-tts output: %w", err)
|
||||
}
|
||||
|
||||
return &audio.SynthResult{
|
||||
Audio: audioBytes,
|
||||
Extension: "mp3",
|
||||
MimeType: "audio/mpeg",
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package elevenlabs
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// client is the shared HTTP client used by the TTS and SFX providers. It
|
||||
// encapsulates the ElevenLabs `xi-api-key` auth scheme + common error handling.
|
||||
type client struct {
|
||||
apiKey string
|
||||
baseURL string
|
||||
timeoutMs int
|
||||
}
|
||||
|
||||
func newClient(apiKey, baseURL string, timeoutMs int) *client {
|
||||
if baseURL == "" {
|
||||
baseURL = "https://api.elevenlabs.io"
|
||||
}
|
||||
if timeoutMs <= 0 {
|
||||
timeoutMs = 30000
|
||||
}
|
||||
return &client{apiKey: apiKey, baseURL: baseURL, timeoutMs: timeoutMs}
|
||||
}
|
||||
|
||||
// postJSON performs a POST with JSON body to {baseURL}/path and returns the
|
||||
// raw response bytes on 200 OK. Non-200 responses surface as errors with the
|
||||
// upstream body appended — matches legacy behavior for debuggability.
|
||||
func (c *client) postJSON(ctx context.Context, path string, body []byte, customTimeout time.Duration) ([]byte, error) {
|
||||
url := strings.TrimRight(c.baseURL, "/") + path
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("xi-api-key", c.apiKey)
|
||||
|
||||
timeout := customTimeout
|
||||
if timeout <= 0 {
|
||||
timeout = time.Duration(c.timeoutMs) * time.Millisecond
|
||||
}
|
||||
hc := &http.Client{Timeout: timeout}
|
||||
resp, err := hc.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("http request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
errBody, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("ElevenLabs API error %d: %s", resp.StatusCode, truncate(errBody, 500))
|
||||
}
|
||||
out, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read response: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func truncate(b []byte, n int) string {
|
||||
if len(b) <= n {
|
||||
return string(b)
|
||||
}
|
||||
return string(b[:n]) + "...(truncated)"
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
// Package elevenlabs implements the ElevenLabs audio API for TTS and SFX.
|
||||
// TTS ships in Phase 1; SFX migrated from internal/tools/create_audio_elevenlabs.go.
|
||||
// Music generation via /v1/music/compose lands in Phase 3.
|
||||
package elevenlabs
|
||||
|
||||
// Config bundles credentials + TTS defaults for ElevenLabs. Shared by the
|
||||
// TTS and SFX providers — SFX ignores Voice/Model fields.
|
||||
type Config struct {
|
||||
APIKey string
|
||||
BaseURL string // default "https://api.elevenlabs.io"
|
||||
VoiceID string // default "pMsXgVXv3BLzUgSXRplE" (TTS only)
|
||||
ModelID string // default "eleven_multilingual_v2" (TTS only)
|
||||
TimeoutMs int // default 30000
|
||||
}
|
||||
|
||||
// defaults fills in blank fields with the values used by the legacy tts package.
|
||||
func (c Config) withDefaults() Config {
|
||||
if c.BaseURL == "" {
|
||||
c.BaseURL = "https://api.elevenlabs.io"
|
||||
}
|
||||
if c.VoiceID == "" {
|
||||
c.VoiceID = "pMsXgVXv3BLzUgSXRplE"
|
||||
}
|
||||
if c.ModelID == "" {
|
||||
c.ModelID = "eleven_multilingual_v2"
|
||||
}
|
||||
if c.TimeoutMs <= 0 {
|
||||
c.TimeoutMs = 30000
|
||||
}
|
||||
return c
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package elevenlabs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/audio"
|
||||
)
|
||||
|
||||
// sfxMaxDurationSeconds is ElevenLabs' upper limit for /v1/sound-generation.
|
||||
const sfxMaxDurationSeconds = 30
|
||||
|
||||
// SFXProvider generates short sound effects via POST /v1/sound-generation.
|
||||
// Migrated from internal/tools/create_audio_elevenlabs.go — byte-for-byte
|
||||
// identical request payload + 60s timeout.
|
||||
type SFXProvider struct {
|
||||
cfg Config
|
||||
c *client
|
||||
}
|
||||
|
||||
// NewSFXProvider returns an ElevenLabs SFX provider.
|
||||
func NewSFXProvider(cfg Config) *SFXProvider {
|
||||
cfg = cfg.withDefaults()
|
||||
return &SFXProvider{
|
||||
cfg: cfg,
|
||||
c: newClient(cfg.APIKey, cfg.BaseURL, cfg.TimeoutMs),
|
||||
}
|
||||
}
|
||||
|
||||
// Name returns the stable provider identifier used by the Manager.
|
||||
func (p *SFXProvider) Name() string { return "elevenlabs" }
|
||||
|
||||
// GenerateSFX produces an MP3 sound effect from opts.Prompt. Duration is
|
||||
// capped at sfxMaxDurationSeconds (ElevenLabs limit).
|
||||
func (p *SFXProvider) GenerateSFX(ctx context.Context, opts audio.SFXOptions) (*audio.AudioResult, error) {
|
||||
duration := opts.Duration
|
||||
if duration > sfxMaxDurationSeconds {
|
||||
duration = sfxMaxDurationSeconds
|
||||
}
|
||||
|
||||
body := map[string]any{
|
||||
"text": opts.Prompt,
|
||||
"output_format": "mp3_44100_128",
|
||||
"prompt_influence": 0.3,
|
||||
}
|
||||
if duration > 0 {
|
||||
body["duration_seconds"] = duration
|
||||
}
|
||||
|
||||
jsonBody, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal sfx request: %w", err)
|
||||
}
|
||||
|
||||
// Sound-generation is bursty — legacy code used a fixed 60s timeout, not
|
||||
// the per-call TimeoutMs. Preserve that exactly.
|
||||
audioBytes, err := p.c.postJSON(ctx, "/v1/sound-generation", jsonBody, 60*time.Second)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("elevenlabs sfx: %w", err)
|
||||
}
|
||||
if len(audioBytes) == 0 {
|
||||
return nil, fmt.Errorf("empty audio response from ElevenLabs")
|
||||
}
|
||||
|
||||
return &audio.AudioResult{
|
||||
Audio: audioBytes,
|
||||
Extension: "mp3",
|
||||
MimeType: "audio/mpeg",
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package elevenlabs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/audio"
|
||||
)
|
||||
|
||||
// TTSProvider synthesizes text via POST /v1/text-to-speech/{voice_id}.
|
||||
// Legacy name: internal/tts.ElevenLabsProvider.
|
||||
type TTSProvider struct {
|
||||
cfg Config
|
||||
c *client
|
||||
}
|
||||
|
||||
// NewTTSProvider returns an ElevenLabs TTS provider backed by the shared client.
|
||||
func NewTTSProvider(cfg Config) *TTSProvider {
|
||||
cfg = cfg.withDefaults()
|
||||
return &TTSProvider{
|
||||
cfg: cfg,
|
||||
c: newClient(cfg.APIKey, cfg.BaseURL, cfg.TimeoutMs),
|
||||
}
|
||||
}
|
||||
|
||||
// Name returns the stable provider identifier used by the Manager.
|
||||
func (p *TTSProvider) Name() string { return "elevenlabs" }
|
||||
|
||||
// Synthesize converts text to audio. Opts.Voice/Opts.Model override the
|
||||
// configured defaults; Opts.Format="opus" switches to Ogg Opus output.
|
||||
func (p *TTSProvider) Synthesize(ctx context.Context, text string, opts audio.TTSOptions) (*audio.SynthResult, error) {
|
||||
voiceID := opts.Voice
|
||||
if voiceID == "" {
|
||||
voiceID = p.cfg.VoiceID
|
||||
}
|
||||
modelID := opts.Model
|
||||
if modelID == "" {
|
||||
modelID = p.cfg.ModelID
|
||||
}
|
||||
|
||||
// Output format: MP3 128kbps default, Opus 64kbps for Telegram voice.
|
||||
outputFormat, ext, mime := "mp3_44100_128", "mp3", "audio/mpeg"
|
||||
if opts.Format == "opus" {
|
||||
outputFormat, ext, mime = "opus_48000_64", "ogg", "audio/ogg"
|
||||
}
|
||||
|
||||
body := map[string]any{
|
||||
"text": text,
|
||||
"model_id": modelID,
|
||||
"voice_settings": map[string]any{
|
||||
"stability": 0.5,
|
||||
"similarity_boost": 0.75,
|
||||
"style": 0.0,
|
||||
"use_speaker_boost": true,
|
||||
},
|
||||
}
|
||||
bodyJSON, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal elevenlabs tts request: %w", err)
|
||||
}
|
||||
|
||||
path := fmt.Sprintf("/v1/text-to-speech/%s?output_format=%s", voiceID, outputFormat)
|
||||
audioBytes, err := p.c.postJSON(ctx, path, bodyJSON, 0)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("elevenlabs tts: %w", err)
|
||||
}
|
||||
|
||||
return &audio.SynthResult{
|
||||
Audio: audioBytes,
|
||||
Extension: ext,
|
||||
MimeType: mime,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package audio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
)
|
||||
|
||||
// Manager orchestrates audio providers across TTS, STT, Music, and SFX
|
||||
// operations. Each op has its own provider map + primary/fallback chain.
|
||||
//
|
||||
// Phase 1 exercises the TTS path end-to-end. STT/Music/SFX maps and chains
|
||||
// are present but empty (providers register in Phase 3/4).
|
||||
type Manager struct {
|
||||
ttsProviders map[string]TTSProvider
|
||||
sttProviders map[string]STTProvider
|
||||
musicProviders map[string]MusicProvider
|
||||
sfxProviders map[string]SFXProvider
|
||||
|
||||
primary string // primary TTS provider
|
||||
sttChain []string // STT fallback order (Phase 4)
|
||||
musicChain []string // Music fallback order (Phase 3)
|
||||
|
||||
auto AutoMode
|
||||
mode Mode
|
||||
maxLength int // max text length before truncation (default 1500)
|
||||
timeoutMs int // provider timeout (default 30000)
|
||||
}
|
||||
|
||||
// ManagerConfig configures the audio manager. Preserved from legacy TTS
|
||||
// package — new STT/Music fields are set via RegisterSTT/RegisterMusic and
|
||||
// (optionally) cfg.Audio in config_audio.go.
|
||||
type ManagerConfig struct {
|
||||
Primary string // primary TTS provider name
|
||||
Auto AutoMode // auto-apply mode (default "off")
|
||||
Mode Mode // "final" or "all" (default "final")
|
||||
MaxLength int // default 1500
|
||||
TimeoutMs int // default 30000
|
||||
}
|
||||
|
||||
// NewManager creates an audio manager with empty provider maps.
|
||||
func NewManager(cfg ManagerConfig) *Manager {
|
||||
m := &Manager{
|
||||
ttsProviders: make(map[string]TTSProvider),
|
||||
sttProviders: make(map[string]STTProvider),
|
||||
musicProviders: make(map[string]MusicProvider),
|
||||
sfxProviders: make(map[string]SFXProvider),
|
||||
primary: cfg.Primary,
|
||||
auto: cfg.Auto,
|
||||
mode: cfg.Mode,
|
||||
maxLength: cfg.MaxLength,
|
||||
timeoutMs: cfg.TimeoutMs,
|
||||
}
|
||||
if m.auto == "" {
|
||||
m.auto = AutoOff
|
||||
}
|
||||
if m.mode == "" {
|
||||
m.mode = ModeFinal
|
||||
}
|
||||
if m.maxLength <= 0 {
|
||||
m.maxLength = 1500
|
||||
}
|
||||
if m.timeoutMs <= 0 {
|
||||
m.timeoutMs = 30000
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// ---- Registration ----
|
||||
|
||||
// RegisterTTS adds a TTS provider. If no primary is set, the first registered
|
||||
// provider becomes primary — matches legacy tts.Manager.RegisterProvider.
|
||||
func (m *Manager) RegisterTTS(p TTSProvider) {
|
||||
m.ttsProviders[p.Name()] = p
|
||||
if m.primary == "" {
|
||||
m.primary = p.Name()
|
||||
}
|
||||
}
|
||||
|
||||
// RegisterProvider is a backward-compat alias for RegisterTTS — lets pre-Phase-1
|
||||
// callers that go through tts.Manager (= audio.Manager via alias) keep working.
|
||||
func (m *Manager) RegisterProvider(p TTSProvider) { m.RegisterTTS(p) }
|
||||
|
||||
// RegisterSTT adds an STT provider (Phase 4).
|
||||
func (m *Manager) RegisterSTT(p STTProvider) {
|
||||
m.sttProviders[p.Name()] = p
|
||||
}
|
||||
|
||||
// RegisterMusic adds a music provider (Phase 3).
|
||||
func (m *Manager) RegisterMusic(p MusicProvider) {
|
||||
m.musicProviders[p.Name()] = p
|
||||
}
|
||||
|
||||
// RegisterSFX adds an SFX provider (Phase 3).
|
||||
func (m *Manager) RegisterSFX(p SFXProvider) {
|
||||
m.sfxProviders[p.Name()] = p
|
||||
}
|
||||
|
||||
// ---- Introspection ----
|
||||
|
||||
// GetProvider returns a TTS provider by name. Preserved from legacy API.
|
||||
func (m *Manager) GetProvider(name string) (TTSProvider, bool) {
|
||||
p, ok := m.ttsProviders[name]
|
||||
return p, ok
|
||||
}
|
||||
|
||||
// PrimaryProvider returns the primary TTS provider name.
|
||||
func (m *Manager) PrimaryProvider() string { return m.primary }
|
||||
|
||||
// AutoMode returns the current auto-apply mode.
|
||||
func (m *Manager) AutoMode() AutoMode { return m.auto }
|
||||
|
||||
// HasProviders reports whether any TTS provider is registered.
|
||||
func (m *Manager) HasProviders() bool { return len(m.ttsProviders) > 0 }
|
||||
|
||||
// ---- TTS dispatch ----
|
||||
|
||||
// Synthesize uses the primary provider.
|
||||
func (m *Manager) Synthesize(ctx context.Context, text string, opts TTSOptions) (*SynthResult, error) {
|
||||
p, ok := m.ttsProviders[m.primary]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("tts provider not found: %s", m.primary)
|
||||
}
|
||||
return p.Synthesize(ctx, text, opts)
|
||||
}
|
||||
|
||||
// SynthesizeWithFallback tries primary first, then any other registered
|
||||
// provider on error. Returns first success or aggregate failure.
|
||||
func (m *Manager) SynthesizeWithFallback(ctx context.Context, text string, opts TTSOptions) (*SynthResult, error) {
|
||||
if p, ok := m.ttsProviders[m.primary]; ok {
|
||||
if result, err := p.Synthesize(ctx, text, opts); err == nil {
|
||||
return result, nil
|
||||
} else {
|
||||
slog.Warn("tts primary provider failed, trying fallback", "provider", m.primary, "error", err)
|
||||
}
|
||||
}
|
||||
for name, p := range m.ttsProviders {
|
||||
if name == m.primary {
|
||||
continue
|
||||
}
|
||||
result, err := p.Synthesize(ctx, text, opts)
|
||||
if err == nil {
|
||||
slog.Info("tts fallback succeeded", "provider", name)
|
||||
return result, nil
|
||||
}
|
||||
slog.Warn("tts fallback provider failed", "provider", name, "error", err)
|
||||
}
|
||||
return nil, fmt.Errorf("all tts providers failed")
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package audio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// MaybeApply inspects auto-mode and conditionally applies TTS to a reply.
|
||||
// Returns (result, true) on success, (nil, false) when auto is disabled, the
|
||||
// reply type is filtered out, content fails validation, or synthesis fails.
|
||||
//
|
||||
// Parameters:
|
||||
// - text: the reply text to potentially convert
|
||||
// - channel: origin channel ("telegram" switches format to opus)
|
||||
// - isVoiceInbound: whether the user's inbound message was voice
|
||||
// - kind: "tool", "block", or "final"
|
||||
func (m *Manager) MaybeApply(ctx context.Context, text, channel string, isVoiceInbound bool, kind string) (*SynthResult, bool) {
|
||||
if m.auto == AutoOff {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// Mode filter: ModeFinal skips tool/block replies.
|
||||
if m.mode == ModeFinal && (kind == "tool" || kind == "block") {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
switch m.auto {
|
||||
case AutoInbound:
|
||||
if !isVoiceInbound {
|
||||
return nil, false
|
||||
}
|
||||
case AutoTagged:
|
||||
if !strings.Contains(text, "[[tts]]") && !strings.Contains(text, "[[tts:") {
|
||||
return nil, false
|
||||
}
|
||||
case AutoAlways:
|
||||
// Always apply.
|
||||
default:
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// Content validation (matches legacy TTS behavior).
|
||||
cleanText := stripMarkdown(text)
|
||||
cleanText = stripTtsDirectives(cleanText)
|
||||
cleanText = strings.TrimSpace(cleanText)
|
||||
|
||||
if len(cleanText) < 10 {
|
||||
return nil, false
|
||||
}
|
||||
if strings.Contains(cleanText, "MEDIA:") {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
if len(cleanText) > m.maxLength {
|
||||
cleanText = cleanText[:m.maxLength] + "..."
|
||||
}
|
||||
|
||||
opts := TTSOptions{}
|
||||
if channel == "telegram" {
|
||||
opts.Format = "opus" // Telegram voice bubbles need opus
|
||||
}
|
||||
|
||||
result, err := m.SynthesizeWithFallback(ctx, cleanText, opts)
|
||||
if err != nil {
|
||||
slog.Warn("tts auto-apply failed", "error", err)
|
||||
return nil, false
|
||||
}
|
||||
return result, true
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package audio_test
|
||||
|
||||
// Manager skeleton smoke test. Red-first: this file fails to compile until
|
||||
// internal/audio is created with Manager, TTSProvider, and the 4 Register*
|
||||
// functions. Once compile passes, tests exercise the minimum surface needed
|
||||
// by phase-01 callers (NewManager, RegisterTTS, HasProviders, zero-value
|
||||
// chains).
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/audio"
|
||||
)
|
||||
|
||||
func TestNewManager_ReturnsNonNil(t *testing.T) {
|
||||
mgr := audio.NewManager(audio.ManagerConfig{Primary: "test"})
|
||||
if mgr == nil {
|
||||
t.Fatal("NewManager returned nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestManager_ZeroValueChains_AreEmpty(t *testing.T) {
|
||||
mgr := audio.NewManager(audio.ManagerConfig{})
|
||||
if mgr.HasProviders() {
|
||||
t.Fatal("fresh Manager should not report providers")
|
||||
}
|
||||
if _, ok := mgr.GetProvider("anything"); ok {
|
||||
t.Fatal("fresh Manager should not resolve providers")
|
||||
}
|
||||
}
|
||||
|
||||
type stubTTS struct{ name string }
|
||||
|
||||
func (s stubTTS) Name() string { return s.name }
|
||||
func (s stubTTS) Synthesize(_ any, _ string, _ audio.TTSOptions) (*audio.SynthResult, error) {
|
||||
return &audio.SynthResult{Extension: "mp3", MimeType: "audio/mpeg"}, nil
|
||||
}
|
||||
|
||||
// NOTE: stubTTS.Synthesize takes `any` for ctx to avoid importing context in
|
||||
// the smoke test — the TTSProvider interface will enforce context.Context via
|
||||
// compile check when we add `var _ audio.TTSProvider = stubTTS{}`.
|
||||
//
|
||||
// Smoke test intentionally omits the `var _ audio.TTSProvider = stubTTS{}`
|
||||
// assertion because stubTTS uses `any` for ctx. Provider-interface conformance
|
||||
// is covered by the alias_test (tts.Provider alias of audio.TTSProvider) and
|
||||
// by production provider packages (elevenlabs, openai, edge, minimax).
|
||||
@@ -0,0 +1,163 @@
|
||||
// Package minimax implements TTS via the MiniMax T2A v2 API.
|
||||
// Docs: https://platform.minimax.io/docs/api-reference/speech-t2a-intro
|
||||
package minimax
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/audio"
|
||||
)
|
||||
|
||||
// Config configures the MiniMax TTS provider.
|
||||
type Config struct {
|
||||
APIKey string
|
||||
GroupID string // required query param
|
||||
APIBase string // default "https://api.minimax.io/v1"
|
||||
Model string // default "speech-02-hd"
|
||||
VoiceID string // default "Wise_Woman"
|
||||
TimeoutMs int
|
||||
}
|
||||
|
||||
// Provider implements audio.TTSProvider via MiniMax /t2a_v2.
|
||||
type Provider struct {
|
||||
apiKey string
|
||||
groupID string
|
||||
apiBase string
|
||||
model string
|
||||
voiceID string
|
||||
timeoutMs int
|
||||
}
|
||||
|
||||
// NewProvider returns a MiniMax TTS provider with defaults applied.
|
||||
func NewProvider(cfg Config) *Provider {
|
||||
p := &Provider{
|
||||
apiKey: cfg.APIKey,
|
||||
groupID: cfg.GroupID,
|
||||
apiBase: cfg.APIBase,
|
||||
model: cfg.Model,
|
||||
voiceID: cfg.VoiceID,
|
||||
timeoutMs: cfg.TimeoutMs,
|
||||
}
|
||||
if p.apiBase == "" {
|
||||
p.apiBase = "https://api.minimax.io/v1"
|
||||
}
|
||||
if p.model == "" {
|
||||
p.model = "speech-02-hd"
|
||||
}
|
||||
if p.voiceID == "" {
|
||||
p.voiceID = "Wise_Woman"
|
||||
}
|
||||
if p.timeoutMs <= 0 {
|
||||
p.timeoutMs = 30000
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// Name returns the stable provider identifier used by the Manager.
|
||||
func (p *Provider) Name() string { return "minimax" }
|
||||
|
||||
// Synthesize calls MiniMax t2a_v2 (non-streaming). MiniMax returns hex-encoded
|
||||
// audio in response.data.audio — we decode before returning.
|
||||
func (p *Provider) Synthesize(ctx context.Context, text string, opts audio.TTSOptions) (*audio.SynthResult, error) {
|
||||
voiceID := opts.Voice
|
||||
if voiceID == "" {
|
||||
voiceID = p.voiceID
|
||||
}
|
||||
model := opts.Model
|
||||
if model == "" {
|
||||
model = p.model
|
||||
}
|
||||
|
||||
audioFormat, ext, mime := "mp3", "mp3", "audio/mpeg"
|
||||
if opts.Format == "opus" || opts.Format == "pcm" || opts.Format == "flac" || opts.Format == "wav" {
|
||||
audioFormat = opts.Format
|
||||
switch opts.Format {
|
||||
case "pcm":
|
||||
ext, mime = "pcm", "audio/pcm"
|
||||
case "flac":
|
||||
ext, mime = "flac", "audio/flac"
|
||||
case "wav":
|
||||
ext, mime = "wav", "audio/wav"
|
||||
}
|
||||
}
|
||||
|
||||
body := map[string]any{
|
||||
"text": text,
|
||||
"model": model,
|
||||
"stream": false,
|
||||
"voice_setting": map[string]any{
|
||||
"voice_id": voiceID,
|
||||
"speed": 1.0,
|
||||
"pitch": 0,
|
||||
},
|
||||
"audio_setting": map[string]any{
|
||||
"format": audioFormat,
|
||||
},
|
||||
}
|
||||
bodyJSON, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal minimax tts request: %w", err)
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("%s/t2a_v2?GroupId=%s", p.apiBase, p.groupID)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(bodyJSON))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create minimax tts request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+p.apiKey)
|
||||
|
||||
hc := &http.Client{Timeout: time.Duration(p.timeoutMs) * time.Millisecond}
|
||||
resp, err := hc.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("minimax tts request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read minimax tts response: %w", err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("minimax tts error %d: %s", resp.StatusCode, string(respBody))
|
||||
}
|
||||
|
||||
var apiResp miniMaxResponse
|
||||
if err := json.Unmarshal(respBody, &apiResp); err != nil {
|
||||
return nil, fmt.Errorf("parse minimax tts response: %w", err)
|
||||
}
|
||||
if apiResp.BaseResp.StatusCode != 0 {
|
||||
return nil, fmt.Errorf("minimax tts api error %d: %s", apiResp.BaseResp.StatusCode, apiResp.BaseResp.StatusMsg)
|
||||
}
|
||||
if apiResp.Data.Audio == "" {
|
||||
return nil, fmt.Errorf("minimax tts returned empty audio")
|
||||
}
|
||||
|
||||
audioBytes, err := hex.DecodeString(apiResp.Data.Audio)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decode minimax tts audio hex: %w", err)
|
||||
}
|
||||
|
||||
return &audio.SynthResult{
|
||||
Audio: audioBytes,
|
||||
Extension: ext,
|
||||
MimeType: mime,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type miniMaxResponse struct {
|
||||
BaseResp struct {
|
||||
StatusCode int `json:"status_code"`
|
||||
StatusMsg string `json:"status_msg"`
|
||||
} `json:"base_resp"`
|
||||
Data struct {
|
||||
Audio string `json:"audio"` // hex-encoded audio bytes
|
||||
} `json:"data"`
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
// Package openai implements the OpenAI audio/speech API for TTS.
|
||||
package openai
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/audio"
|
||||
)
|
||||
|
||||
// Config bundles credentials + TTS defaults for OpenAI.
|
||||
type Config struct {
|
||||
APIKey string
|
||||
APIBase string // default "https://api.openai.com/v1"
|
||||
Model string // default "gpt-4o-mini-tts"
|
||||
Voice string // default "alloy"
|
||||
TimeoutMs int // default 30000
|
||||
}
|
||||
|
||||
// Provider implements audio.TTSProvider for OpenAI.
|
||||
type Provider struct {
|
||||
apiKey string
|
||||
apiBase string
|
||||
model string
|
||||
voice string
|
||||
timeoutMs int
|
||||
}
|
||||
|
||||
// NewProvider constructs an OpenAI TTS provider with defaults applied.
|
||||
func NewProvider(cfg Config) *Provider {
|
||||
p := &Provider{
|
||||
apiKey: cfg.APIKey,
|
||||
apiBase: cfg.APIBase,
|
||||
model: cfg.Model,
|
||||
voice: cfg.Voice,
|
||||
timeoutMs: cfg.TimeoutMs,
|
||||
}
|
||||
if p.apiBase == "" {
|
||||
p.apiBase = "https://api.openai.com/v1"
|
||||
}
|
||||
if p.model == "" {
|
||||
p.model = "gpt-4o-mini-tts"
|
||||
}
|
||||
if p.voice == "" {
|
||||
p.voice = "alloy"
|
||||
}
|
||||
if p.timeoutMs <= 0 {
|
||||
p.timeoutMs = 30000
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// Name returns the stable provider identifier used by the Manager.
|
||||
func (p *Provider) Name() string { return "openai" }
|
||||
|
||||
// Synthesize calls POST {apiBase}/audio/speech.
|
||||
func (p *Provider) Synthesize(ctx context.Context, text string, opts audio.TTSOptions) (*audio.SynthResult, error) {
|
||||
voice := opts.Voice
|
||||
if voice == "" {
|
||||
voice = p.voice
|
||||
}
|
||||
model := opts.Model
|
||||
if model == "" {
|
||||
model = p.model
|
||||
}
|
||||
format := opts.Format
|
||||
if format == "" {
|
||||
format = "mp3"
|
||||
}
|
||||
|
||||
body := map[string]any{
|
||||
"model": model,
|
||||
"input": text,
|
||||
"voice": voice,
|
||||
"response_format": format,
|
||||
}
|
||||
bodyJSON, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal openai tts request: %w", err)
|
||||
}
|
||||
|
||||
url := p.apiBase + "/audio/speech"
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(bodyJSON))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create openai tts request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+p.apiKey)
|
||||
|
||||
hc := &http.Client{Timeout: time.Duration(p.timeoutMs) * time.Millisecond}
|
||||
resp, err := hc.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("openai tts request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
errBody, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("openai tts error %d: %s", resp.StatusCode, string(errBody))
|
||||
}
|
||||
audioBytes, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read openai tts response: %w", err)
|
||||
}
|
||||
|
||||
ext, mime := format, "audio/mpeg"
|
||||
switch format {
|
||||
case "opus":
|
||||
ext, mime = "ogg", "audio/ogg"
|
||||
case "mp3":
|
||||
mime = "audio/mpeg"
|
||||
}
|
||||
|
||||
return &audio.SynthResult{
|
||||
Audio: audioBytes,
|
||||
Extension: ext,
|
||||
MimeType: mime,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package audio
|
||||
|
||||
import "regexp"
|
||||
|
||||
// stripMarkdown removes common markdown formatting so TTS reads prose, not
|
||||
// syntax characters. Preserves inner text of bold/italic/inline code/links.
|
||||
func stripMarkdown(text string) string {
|
||||
text = regexp.MustCompile("(?s)```[^`]*```").ReplaceAllString(text, "")
|
||||
text = regexp.MustCompile("`([^`]+)`").ReplaceAllString(text, "$1")
|
||||
text = regexp.MustCompile(`\*\*([^*]+)\*\*`).ReplaceAllString(text, "$1")
|
||||
text = regexp.MustCompile(`\*([^*]+)\*`).ReplaceAllString(text, "$1")
|
||||
text = regexp.MustCompile(`__([^_]+)__`).ReplaceAllString(text, "$1")
|
||||
text = regexp.MustCompile(`_([^_]+)_`).ReplaceAllString(text, "$1")
|
||||
text = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`).ReplaceAllString(text, "$1")
|
||||
text = regexp.MustCompile(`(?m)^#+\s+`).ReplaceAllString(text, "")
|
||||
return text
|
||||
}
|
||||
|
||||
// stripTtsDirectives removes [[tts...]] markup from text.
|
||||
// `[[tts:text]]...[[/tts:text]]` blocks keep their inner content.
|
||||
// Bare `[[tts]]` and `[[tts:something]]` tags are removed entirely.
|
||||
func stripTtsDirectives(text string) string {
|
||||
text = regexp.MustCompile(`(?s)\[\[tts:text\]\](.*?)\[\[/tts:text\]\]`).ReplaceAllString(text, "$1")
|
||||
text = regexp.MustCompile(`\[\[tts(?::[^\]]*)?\]\]`).ReplaceAllString(text, "")
|
||||
return text
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
// Package audio unifies TTS, STT, Music, and SFX generation under a single
|
||||
// Manager. Replaces internal/tts (surface preserved via internal/tts/alias.go
|
||||
// backward-compat layer).
|
||||
//
|
||||
// Phase 1 delivers TTS parity; STT/Music/SFX interfaces ship as stubs —
|
||||
// implementations land in Phase 3 (Music/SFX) and Phase 4 (STT).
|
||||
package audio
|
||||
|
||||
import "context"
|
||||
|
||||
// ---- TTS (implemented Phase 1) ----
|
||||
|
||||
// TTSProvider synthesizes text into audio bytes.
|
||||
type TTSProvider interface {
|
||||
Name() string
|
||||
Synthesize(ctx context.Context, text string, opts TTSOptions) (*SynthResult, error)
|
||||
}
|
||||
|
||||
// TTSOptions controls TTS synthesis parameters.
|
||||
type TTSOptions struct {
|
||||
Voice string // provider-specific voice ID
|
||||
Model string // provider-specific model ID
|
||||
Format string // output format: "mp3", "opus", etc.
|
||||
}
|
||||
|
||||
// SynthResult is the output of a TTS synthesis call.
|
||||
type SynthResult struct {
|
||||
Audio []byte // raw audio bytes
|
||||
Extension string // file extension without dot: "mp3", "opus", "ogg"
|
||||
MimeType string // e.g. "audio/mpeg", "audio/ogg"
|
||||
}
|
||||
|
||||
// AutoMode controls when TTS is automatically applied to replies.
|
||||
type AutoMode string
|
||||
|
||||
const (
|
||||
AutoOff AutoMode = "off" // Disabled
|
||||
AutoAlways AutoMode = "always" // Apply to all eligible replies
|
||||
AutoInbound AutoMode = "inbound" // Only if user sent audio/voice
|
||||
AutoTagged AutoMode = "tagged" // Only if reply contains [[tts]] directive
|
||||
)
|
||||
|
||||
// Mode controls which reply kinds get TTS.
|
||||
type Mode string
|
||||
|
||||
const (
|
||||
ModeFinal Mode = "final" // Only final replies (default)
|
||||
ModeAll Mode = "all" // All replies including tool/block
|
||||
)
|
||||
|
||||
// ---- STT (stubs — implementations land in Phase 4) ----
|
||||
|
||||
// STTProvider transcribes audio bytes to text.
|
||||
type STTProvider interface {
|
||||
Name() string
|
||||
Transcribe(ctx context.Context, in STTInput, opts STTOptions) (*TranscriptResult, error)
|
||||
}
|
||||
|
||||
// STTInput is the audio to transcribe. At most one of Audio/FilePath is set.
|
||||
type STTInput struct {
|
||||
Audio []byte // raw audio bytes (in-memory)
|
||||
FilePath string // path on disk (used by proxy_stt)
|
||||
MimeType string // e.g. "audio/ogg"
|
||||
}
|
||||
|
||||
// STTOptions tunes transcription.
|
||||
type STTOptions struct {
|
||||
Language string // BCP-47 hint, empty = auto-detect
|
||||
Model string // provider-specific model ID
|
||||
}
|
||||
|
||||
// TranscriptResult is the output of transcription.
|
||||
type TranscriptResult struct {
|
||||
Text string
|
||||
Language string // detected or hinted language
|
||||
}
|
||||
|
||||
// ---- Music (stubs — implementations land in Phase 3) ----
|
||||
|
||||
// MusicProvider generates music from prompt + optional lyrics.
|
||||
type MusicProvider interface {
|
||||
Name() string
|
||||
GenerateMusic(ctx context.Context, opts MusicOptions) (*AudioResult, error)
|
||||
}
|
||||
|
||||
// MusicOptions controls music generation.
|
||||
type MusicOptions struct {
|
||||
Prompt string
|
||||
Lyrics string
|
||||
Duration int // seconds
|
||||
}
|
||||
|
||||
// ---- SFX (stubs — implementations land in Phase 3) ----
|
||||
|
||||
// SFXProvider generates short sound effects from a prompt.
|
||||
type SFXProvider interface {
|
||||
Name() string
|
||||
GenerateSFX(ctx context.Context, opts SFXOptions) (*AudioResult, error)
|
||||
}
|
||||
|
||||
// SFXOptions controls SFX generation.
|
||||
type SFXOptions struct {
|
||||
Prompt string
|
||||
Duration int // seconds (provider may cap)
|
||||
}
|
||||
|
||||
// AudioResult is the shared output of music/SFX generation.
|
||||
type AudioResult struct {
|
||||
Audio []byte
|
||||
Extension string
|
||||
MimeType string
|
||||
}
|
||||
Reference in new issue
Block a user