diff --git a/cmd/gateway_agents.go b/cmd/gateway_agents.go index 2efec76a..f56febc7 100644 --- a/cmd/gateway_agents.go +++ b/cmd/gateway_agents.go @@ -7,6 +7,7 @@ import ( "github.com/nextlevelbuilder/goclaw/internal/audio/elevenlabs" geminiaudio "github.com/nextlevelbuilder/goclaw/internal/audio/gemini" minimaxaudio "github.com/nextlevelbuilder/goclaw/internal/audio/minimax" + "github.com/nextlevelbuilder/goclaw/internal/audio/openaicompat" "github.com/nextlevelbuilder/goclaw/internal/bus" "github.com/nextlevelbuilder/goclaw/internal/config" "github.com/nextlevelbuilder/goclaw/internal/memory" @@ -273,6 +274,26 @@ func setupTTS(cfg *config.Config) *tts.Manager { })) } + // OpenAI-compatible self-hosted endpoint. api_base is the enable switch — + // unlike the vendor providers there is no API key to gate on, since these + // endpoints commonly have no auth. + if base := ttsCfg.OpenAICompat.APIBase; base != "" { + provider, err := openaicompat.NewTTSProvider(openaicompat.Config{ + APIBase: base, + APIKey: ttsCfg.OpenAICompat.APIKey, + TTSModel: ttsCfg.OpenAICompat.Model, + TTSVoice: ttsCfg.OpenAICompat.Voice, + TTSFormat: ttsCfg.OpenAICompat.Format, + TimeoutMs: ttsCfg.TimeoutMs, + }) + if err != nil { + slog.Warn("audio.tts: openai_compat not registered", "error", err) + } else { + mgr.RegisterProvider(provider) + slog.Info("audio.tts: openai_compat registered", "api_base", base) + } + } + if key := ttsCfg.ElevenLabs.APIKey; key != "" { mgr.RegisterProvider(tts.NewElevenLabsProvider(tts.ElevenLabsConfig{ APIKey: key, @@ -357,6 +378,29 @@ func setupAudioExtras(cfg *config.Config, mgr *tts.Manager) { } } + // STT chain. Built from what actually registered, so Transcribe never walks + // a name it will only skip with a warning. "proxy" is always appended: it is + // registered later, per channel, by BridgeLegacySTT. + var sttChain []string + + // OpenAI-compatible self-hosted endpoint, first when present: it is an + // explicit operator choice, and it keeps audio on the local network. + if base := cfg.Tts.OpenAICompat.APIBase; base != "" { + provider, err := openaicompat.NewSTTProvider(openaicompat.Config{ + APIBase: base, + APIKey: cfg.Tts.OpenAICompat.APIKey, + STTModel: cfg.Tts.OpenAICompat.STTModel, + TimeoutMs: cfg.Tts.TimeoutMs, + }) + if err != nil { + slog.Warn("audio.stt: openai_compat not registered", "error", err) + } else { + mgr.RegisterSTT(provider) + sttChain = append(sttChain, provider.Name()) + slog.Info("audio.stt: openai_compat registered", "api_base", base) + } + } + // ElevenLabs STT (Scribe v2) — reuse TTS credentials. Registered as tenant-scope // default; per-request tenant override lands via builtin_tools[stt] in Phase 5 // channel migration. Legacy per-channel STTProxyURL is bridged separately. @@ -365,7 +409,13 @@ func setupAudioExtras(cfg *config.Config, mgr *tts.Manager) { APIKey: ellKey, BaseURL: ellBase, })) - mgr.SetSTTChain([]string{"elevenlabs", "proxy"}) + sttChain = append(sttChain, "elevenlabs") slog.Info("audio.stt: elevenlabs registered") } + + if len(sttChain) > 0 { + sttChain = append(sttChain, "proxy") + mgr.SetSTTChain(sttChain) + slog.Info("audio.stt: chain configured", "chain", sttChain) + } } diff --git a/internal/audio/openaicompat/config.go b/internal/audio/openaicompat/config.go new file mode 100644 index 00000000..4dcb3a3e --- /dev/null +++ b/internal/audio/openaicompat/config.go @@ -0,0 +1,74 @@ +// Package openaicompat implements the OpenAI audio API (speech + transcriptions) +// against any OpenAI-compatible endpoint — gpu-manager, Speaches, vLLM, LocalAI, +// llama.cpp, Ollama. +// +// It is deliberately distinct from the openai package. Both speak the same wire +// format, but "openai" means api.openai.com semantics: a fixed voice set +// (alloy, echo, ...), an mp3 default, and a base URL that falls back to the +// public API. A self-hosted engine shares none of that — its voices are its own +// (e.g. Piper's "fr_FR-gilles-low"), its formats are whatever it built in, and +// there is no sane default host. +// +// Keeping them separate matters beyond tidiness: audio.IsVoiceCompatible applies +// OpenAI's voice allowlist to any provider named "openai" and silently +// substitutes the default voice for anything outside it. Routing a self-hosted +// engine through that name turns a valid local voice ID into "alloy" with no +// error. Providers without validation rules pass through untouched, which is the +// correct behaviour here. +package openaicompat + +import "errors" + +// ErrAPIBaseRequired is returned by the constructors when APIBase is empty. +// Unlike the openai package there is no public endpoint to fall back to, and +// defaulting to api.openai.com would send self-hosted traffic to a vendor. +var ErrAPIBaseRequired = errors.New("openai_compat: api_base is required") + +// providerName is the stable identifier reported to the audio Manager and used +// in chain configuration. It must not be "openai" — see the package doc. +const providerName = "openai_compat" + +// Default tuning. Formats intentionally follow the OpenAI wire defaults; an +// endpoint that cannot encode mp3 is configured with Format: "wav". +const ( + defaultTTSFormat = "mp3" + defaultTimeoutMs = 30000 +) + +// Config bundles the endpoint, credentials and defaults for both the TTS and +// STT providers. APIBase is required; everything else has a default or is +// resolved per-request. +type Config struct { + // APIBase is the OpenAI-compatible root, including any version prefix — + // e.g. "http://gpu-manager:8080/v1". Required. + APIBase string + // APIKey is sent as a Bearer token when non-empty. Self-hosted endpoints + // frequently have no auth, so an empty key omits the header entirely + // rather than sending "Bearer ". + APIKey string + // TTSModel is the default model for synthesis. Optional: some engines + // resolve the model from the requested voice instead. + TTSModel string + // TTSVoice is the default voice for synthesis. Optional. + TTSVoice string + // TTSFormat is the default response_format for synthesis. Defaults to + // "mp3" to match OpenAI. Set to "wav" or "pcm" for engines without an + // encoder. + TTSFormat string + // STTModel is the default model for transcription. Optional. + STTModel string + // TimeoutMs is the per-request timeout. Defaults to 30000. + TimeoutMs int +} + +// withDefaults returns a copy with zero-valued optional fields filled in. +// APIBase is not defaulted — its absence is an error, not a fallback. +func (c Config) withDefaults() Config { + if c.TTSFormat == "" { + c.TTSFormat = defaultTTSFormat + } + if c.TimeoutMs <= 0 { + c.TimeoutMs = defaultTimeoutMs + } + return c +} diff --git a/internal/audio/openaicompat/stt.go b/internal/audio/openaicompat/stt.go new file mode 100644 index 00000000..7aa04854 --- /dev/null +++ b/internal/audio/openaicompat/stt.go @@ -0,0 +1,211 @@ +package openaicompat + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/nextlevelbuilder/goclaw/internal/audio" +) + +const ( + // sttMaxBytes matches the OpenAI upload limit, which compatible engines + // generally adopt. Enforced before the request so an oversized file fails + // locally instead of after a full upload. + sttMaxBytes = 25 << 20 // 25 MB + // sttDefaultTimeout is the floor for transcription, which is slower than + // synthesis and routinely exceeds the shared 30 s default. + sttDefaultTimeout = 120 * time.Second +) + +// STTProvider transcribes audio via an OpenAI-compatible +// POST /audio/transcriptions. +type STTProvider struct { + cfg Config +} + +// NewSTTProvider returns an STT provider for the configured endpoint. +// Returns ErrAPIBaseRequired when cfg.APIBase is empty. +func NewSTTProvider(cfg Config) (*STTProvider, error) { + if strings.TrimSpace(cfg.APIBase) == "" { + return nil, ErrAPIBaseRequired + } + return &STTProvider{cfg: cfg.withDefaults()}, nil +} + +// Name returns the stable provider identifier. +func (p *STTProvider) Name() string { return providerName } + +// Transcribe converts audio to text. +// +// The multipart filename is significant, not cosmetic: several compatible +// engines type the audio by its extension rather than by Content-Type, and +// reject an upload whose name carries no format they recognise. The filename is +// therefore derived from STTInput.Filename, then the base of FilePath, then the +// MIME type — never left blank. +func (p *STTProvider) Transcribe(ctx context.Context, in audio.STTInput, opts audio.STTOptions) (*audio.TranscriptResult, error) { + body, contentType, err := p.buildForm(in, opts) + if err != nil { + return nil, err + } + + url := strings.TrimRight(p.cfg.APIBase, "/") + "/audio/transcriptions" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, body) + if err != nil { + return nil, fmt.Errorf("openai_compat stt: create request: %w", err) + } + req.Header.Set("Content-Type", contentType) + if p.cfg.APIKey != "" { + req.Header.Set("Authorization", "Bearer "+p.cfg.APIKey) + } + + timeout := sttDefaultTimeout + if opts.TimeoutMs > 0 { + timeout = time.Duration(opts.TimeoutMs) * time.Millisecond + } + hc := &http.Client{Timeout: timeout} + resp, err := hc.Do(req) + if err != nil { + return nil, fmt.Errorf("openai_compat stt: http request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + errBody, _ := io.ReadAll(io.LimitReader(resp.Body, errBodyLimit)) + return nil, fmt.Errorf("openai_compat stt: endpoint error %d: %s", resp.StatusCode, strings.TrimSpace(string(errBody))) + } + + // verbose_json is a superset of json; language and duration are absent from + // a plain json response and stay zero-valued, which the Manager tolerates. + var result struct { + Text string `json:"text"` + Language string `json:"language"` + Duration float64 `json:"duration"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("openai_compat stt: parse response: %w", err) + } + + language := result.Language + if language == "" { + language = opts.Language + } + return &audio.TranscriptResult{ + Text: result.Text, + Language: language, + Duration: result.Duration, + Provider: providerName, + }, nil +} + +// buildForm assembles the multipart body, reading the audio from FilePath or +// Bytes. Returns the body and its Content-Type header value. +func (p *STTProvider) buildForm(in audio.STTInput, opts audio.STTOptions) (io.Reader, string, error) { + var buf bytes.Buffer + mw := multipart.NewWriter(&buf) + + if model := firstNonEmpty(opts.ModelID, p.cfg.STTModel); model != "" { + if err := mw.WriteField("model", model); err != nil { + return nil, "", fmt.Errorf("openai_compat stt: write model field: %w", err) + } + } + if opts.Language != "" { + if err := mw.WriteField("language", opts.Language); err != nil { + return nil, "", fmt.Errorf("openai_compat stt: write language field: %w", err) + } + } + if err := mw.WriteField("response_format", "json"); err != nil { + return nil, "", fmt.Errorf("openai_compat stt: write response_format field: %w", err) + } + + fw, err := mw.CreateFormFile("file", sttFilename(in)) + if err != nil { + return nil, "", fmt.Errorf("openai_compat stt: create form file: %w", err) + } + if err := writeAudio(fw, in); err != nil { + return nil, "", err + } + if err := mw.Close(); err != nil { + return nil, "", fmt.Errorf("openai_compat stt: close multipart writer: %w", err) + } + return &buf, mw.FormDataContentType(), nil +} + +// writeAudio copies the input audio into dst, enforcing the size cap. +func writeAudio(dst io.Writer, in audio.STTInput) error { + if in.FilePath != "" { + info, err := os.Stat(in.FilePath) + if err != nil { + return fmt.Errorf("openai_compat stt: stat file: %w", err) + } + if info.Size() > sttMaxBytes { + return fmt.Errorf("openai_compat stt: file too large (%d bytes, max %d)", info.Size(), sttMaxBytes) + } + f, err := os.Open(in.FilePath) + if err != nil { + return fmt.Errorf("openai_compat stt: open file: %w", err) + } + defer f.Close() + if _, err := io.Copy(dst, f); err != nil { + return fmt.Errorf("openai_compat stt: read file: %w", err) + } + return nil + } + + if len(in.Bytes) == 0 { + return fmt.Errorf("openai_compat stt: neither FilePath nor Bytes provided") + } + if len(in.Bytes) > sttMaxBytes { + return fmt.Errorf("openai_compat stt: audio too large (%d bytes, max %d)", len(in.Bytes), sttMaxBytes) + } + if _, err := dst.Write(in.Bytes); err != nil { + return fmt.Errorf("openai_compat stt: write audio bytes: %w", err) + } + return nil +} + +// sttFilename derives the multipart filename. See Transcribe for why a blank +// name is not an option. +func sttFilename(in audio.STTInput) string { + if in.Filename != "" { + return in.Filename + } + if in.FilePath != "" { + return filepath.Base(in.FilePath) + } + return "audio" + extFromMime(in.MimeType) +} + +// extFromMime maps a MIME type to a file extension, including the leading dot. +func extFromMime(mime string) string { + // Strip codec parameters, e.g. "audio/ogg; codecs=opus". + if idx := strings.IndexByte(mime, ';'); idx >= 0 { + mime = mime[:idx] + } + switch strings.TrimSpace(mime) { + case "audio/ogg", "audio/opus": + return ".ogg" + case "audio/mpeg", "audio/mp3": + return ".mp3" + case "audio/wav", "audio/wave", "audio/x-wav": + return ".wav" + case "audio/pcm", "audio/l16": + return ".pcm" + case "audio/mp4", "audio/m4a", "audio/x-m4a": + return ".m4a" + case "audio/webm": + return ".webm" + case "audio/flac", "audio/x-flac": + return ".flac" + default: + return ".bin" + } +} diff --git a/internal/audio/openaicompat/stt_test.go b/internal/audio/openaicompat/stt_test.go new file mode 100644 index 00000000..7654caea --- /dev/null +++ b/internal/audio/openaicompat/stt_test.go @@ -0,0 +1,267 @@ +package openaicompat_test + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/nextlevelbuilder/goclaw/internal/audio" + "github.com/nextlevelbuilder/goclaw/internal/audio/openaicompat" +) + +// sttForm captures what the fake endpoint received. +type sttForm struct { + path string + auth string + filename string + fileData string + model string + language string + format string +} + +// newSTTServer returns a fake OpenAI-compatible transcription endpoint that +// records the multipart form and replies with body. +func newSTTServer(t *testing.T, got *sttForm, status int, body string) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got.path = r.URL.Path + got.auth = r.Header.Get("Authorization") + require.NoError(t, r.ParseMultipartForm(32<<20)) + got.model = r.FormValue("model") + got.language = r.FormValue("language") + got.format = r.FormValue("response_format") + + file, header, err := r.FormFile("file") + if err == nil { + defer file.Close() + got.filename = header.Filename + data, err := io.ReadAll(file) + require.NoError(t, err) + got.fileData = string(data) + } + + w.WriteHeader(status) + _, _ = io.WriteString(w, body) + })) + t.Cleanup(func() { srv.Close() }) + return srv +} + +func TestNewSTTProviderRequiresAPIBase(t *testing.T) { + _, err := openaicompat.NewSTTProvider(openaicompat.Config{}) + require.ErrorIs(t, err, openaicompat.ErrAPIBaseRequired) +} + +func TestSTTProviderName(t *testing.T) { + provider, err := openaicompat.NewSTTProvider(openaicompat.Config{APIBase: "http://x/v1"}) + require.NoError(t, err) + assert.Equal(t, "openai_compat", provider.Name()) +} + +func TestSTTTranscribeFromBytes(t *testing.T) { + var got sttForm + srv := newSTTServer(t, &got, http.StatusOK, `{"text":"bonjour le monde","language":"fr","duration":1.5}`) + + provider, err := openaicompat.NewSTTProvider(openaicompat.Config{ + APIBase: srv.URL + "/v1", + APIKey: "secret", + STTModel: "whisper-1", + }) + require.NoError(t, err) + + result, err := provider.Transcribe(context.Background(), + audio.STTInput{Bytes: []byte("RIFFfake"), MimeType: "audio/wav", Filename: "utterance.wav"}, + audio.STTOptions{Language: "fr"}, + ) + require.NoError(t, err) + + assert.Equal(t, "/v1/audio/transcriptions", got.path) + assert.Equal(t, "Bearer secret", got.auth) + assert.Equal(t, "utterance.wav", got.filename) + assert.Equal(t, "RIFFfake", got.fileData) + assert.Equal(t, "whisper-1", got.model) + assert.Equal(t, "fr", got.language) + assert.Equal(t, "json", got.format) + + assert.Equal(t, "bonjour le monde", result.Text) + assert.Equal(t, "fr", result.Language) + assert.InEpsilon(t, 1.5, result.Duration, 0.0001) + assert.Equal(t, "openai_compat", result.Provider) +} + +func TestSTTTranscribeFromFilePath(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "voice-note.ogg") + require.NoError(t, os.WriteFile(path, []byte("OggS-fake"), 0o600)) + + var got sttForm + srv := newSTTServer(t, &got, http.StatusOK, `{"text":"salut"}`) + + provider, err := openaicompat.NewSTTProvider(openaicompat.Config{APIBase: srv.URL}) + require.NoError(t, err) + + result, err := provider.Transcribe(context.Background(), + audio.STTInput{FilePath: path, MimeType: "audio/ogg"}, + audio.STTOptions{}, + ) + require.NoError(t, err) + + assert.Equal(t, "voice-note.ogg", got.filename, "filename falls back to the base of FilePath") + assert.Equal(t, "OggS-fake", got.fileData) + assert.Equal(t, "salut", result.Text) +} + +// TestSTTFilenameIsNeverBlank guards the extension-typing requirement: some +// compatible engines (gpu-manager among them) infer the audio format from the +// filename and reject an upload they cannot type. +func TestSTTFilenameIsNeverBlank(t *testing.T) { + tests := []struct { + name string + in audio.STTInput + wantName string + }{ + { + name: "explicit filename wins", + in: audio.STTInput{Bytes: []byte("x"), Filename: "utterance.wav", MimeType: "audio/ogg"}, + wantName: "utterance.wav", + }, + { + name: "derived from wav mime", + in: audio.STTInput{Bytes: []byte("x"), MimeType: "audio/wav"}, + wantName: "audio.wav", + }, + { + name: "derived from ogg mime", + in: audio.STTInput{Bytes: []byte("x"), MimeType: "audio/ogg"}, + wantName: "audio.ogg", + }, + { + name: "codec parameters stripped", + in: audio.STTInput{Bytes: []byte("x"), MimeType: "audio/ogg; codecs=opus"}, + wantName: "audio.ogg", + }, + { + name: "mpeg mime", + in: audio.STTInput{Bytes: []byte("x"), MimeType: "audio/mpeg"}, + wantName: "audio.mp3", + }, + { + name: "unknown mime still carries an extension", + in: audio.STTInput{Bytes: []byte("x"), MimeType: "audio/weird"}, + wantName: "audio.bin", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var got sttForm + srv := newSTTServer(t, &got, http.StatusOK, `{"text":"ok"}`) + + provider, err := openaicompat.NewSTTProvider(openaicompat.Config{APIBase: srv.URL}) + require.NoError(t, err) + + _, err = provider.Transcribe(context.Background(), tc.in, audio.STTOptions{}) + require.NoError(t, err) + assert.Equal(t, tc.wantName, got.filename) + }) + } +} + +func TestSTTOmitsModelWhenUnset(t *testing.T) { + var got sttForm + srv := newSTTServer(t, &got, http.StatusOK, `{"text":"ok"}`) + + provider, err := openaicompat.NewSTTProvider(openaicompat.Config{APIBase: srv.URL}) + require.NoError(t, err) + + _, err = provider.Transcribe(context.Background(), + audio.STTInput{Bytes: []byte("x"), MimeType: "audio/wav"}, audio.STTOptions{}) + require.NoError(t, err) + assert.Empty(t, got.model, "an unset model must be omitted so the endpoint picks its own") +} + +func TestSTTOptionsModelOverridesConfig(t *testing.T) { + var got sttForm + srv := newSTTServer(t, &got, http.StatusOK, `{"text":"ok"}`) + + provider, err := openaicompat.NewSTTProvider(openaicompat.Config{APIBase: srv.URL, STTModel: "config-model"}) + require.NoError(t, err) + + _, err = provider.Transcribe(context.Background(), + audio.STTInput{Bytes: []byte("x"), MimeType: "audio/wav"}, + audio.STTOptions{ModelID: "opts-model"}) + require.NoError(t, err) + assert.Equal(t, "opts-model", got.model) +} + +// TestSTTLanguageFallsBackToHint covers a plain (non-verbose) json response, +// which carries no language field. +func TestSTTLanguageFallsBackToHint(t *testing.T) { + var got sttForm + srv := newSTTServer(t, &got, http.StatusOK, `{"text":"bonjour"}`) + + provider, err := openaicompat.NewSTTProvider(openaicompat.Config{APIBase: srv.URL}) + require.NoError(t, err) + + result, err := provider.Transcribe(context.Background(), + audio.STTInput{Bytes: []byte("x"), MimeType: "audio/wav"}, + audio.STTOptions{Language: "fr"}) + require.NoError(t, err) + assert.Equal(t, "fr", result.Language) +} + +func TestSTTErrorStatus(t *testing.T) { + var got sttForm + srv := newSTTServer(t, &got, http.StatusUnsupportedMediaType, `{"error":"unsupported audio format"}`) + + provider, err := openaicompat.NewSTTProvider(openaicompat.Config{APIBase: srv.URL}) + require.NoError(t, err) + + _, err = provider.Transcribe(context.Background(), + audio.STTInput{Bytes: []byte("x"), MimeType: "audio/ogg"}, audio.STTOptions{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "415") + assert.Contains(t, err.Error(), "unsupported audio format") +} + +func TestSTTRejectsEmptyInput(t *testing.T) { + provider, err := openaicompat.NewSTTProvider(openaicompat.Config{APIBase: "http://x/v1"}) + require.NoError(t, err) + + _, err = provider.Transcribe(context.Background(), audio.STTInput{}, audio.STTOptions{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "neither FilePath nor Bytes") +} + +func TestSTTRejectsOversizedBytes(t *testing.T) { + provider, err := openaicompat.NewSTTProvider(openaicompat.Config{APIBase: "http://x/v1"}) + require.NoError(t, err) + + _, err = provider.Transcribe(context.Background(), + audio.STTInput{Bytes: make([]byte, (25<<20)+1), MimeType: "audio/wav"}, + audio.STTOptions{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "too large") +} + +func TestSTTRejectsOversizedFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "big.wav") + require.NoError(t, os.WriteFile(path, make([]byte, (25<<20)+1), 0o600)) + + provider, err := openaicompat.NewSTTProvider(openaicompat.Config{APIBase: "http://x/v1"}) + require.NoError(t, err) + + _, err = provider.Transcribe(context.Background(), + audio.STTInput{FilePath: path, MimeType: "audio/wav"}, audio.STTOptions{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "too large") +} diff --git a/internal/audio/openaicompat/tts.go b/internal/audio/openaicompat/tts.go new file mode 100644 index 00000000..7c583c50 --- /dev/null +++ b/internal/audio/openaicompat/tts.go @@ -0,0 +1,170 @@ +package openaicompat + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/nextlevelbuilder/goclaw/internal/audio" +) + +// errBodyLimit caps how much of an error response body is read into a message. +const errBodyLimit = 512 + +// TTSProvider synthesizes speech via an OpenAI-compatible POST /audio/speech. +type TTSProvider struct { + cfg Config +} + +// NewTTSProvider returns a TTS provider for the configured endpoint. +// Returns ErrAPIBaseRequired when cfg.APIBase is empty. +func NewTTSProvider(cfg Config) (*TTSProvider, error) { + if strings.TrimSpace(cfg.APIBase) == "" { + return nil, ErrAPIBaseRequired + } + return &TTSProvider{cfg: cfg.withDefaults()}, nil +} + +// Name returns the stable provider identifier. +func (p *TTSProvider) Name() string { return providerName } + +// Synthesize calls POST {api_base}/audio/speech and returns the audio bytes. +// +// Resolution order for voice, model and format is: per-request opts, then +// opts.Params, then the provider config. Voice and model are omitted from the +// request when they resolve empty — some engines pick the model from the voice, +// and sending an empty string would override that choice rather than defer to it. +func (p *TTSProvider) Synthesize(ctx context.Context, text string, opts audio.TTSOptions) (*audio.SynthResult, error) { + format := firstNonEmpty(opts.Format, paramString(opts.Params, "response_format"), p.cfg.TTSFormat) + voice := firstNonEmpty(opts.Voice, p.cfg.TTSVoice) + model := firstNonEmpty(opts.Model, p.cfg.TTSModel) + + body := map[string]any{ + "input": text, + "response_format": format, + } + if voice != "" { + body["voice"] = voice + } + if model != "" { + body["model"] = model + } + if speed := paramFloat(opts.Params, "speed"); speed > 0 { + body["speed"] = speed + } + + bodyJSON, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("openai_compat tts: marshal request: %w", err) + } + + url := strings.TrimRight(p.cfg.APIBase, "/") + "/audio/speech" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(bodyJSON)) + if err != nil { + return nil, fmt.Errorf("openai_compat tts: create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + if p.cfg.APIKey != "" { + req.Header.Set("Authorization", "Bearer "+p.cfg.APIKey) + } + + hc := &http.Client{Timeout: time.Duration(p.cfg.TimeoutMs) * time.Millisecond} + resp, err := hc.Do(req) + if err != nil { + return nil, fmt.Errorf("openai_compat tts: http request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + errBody, _ := io.ReadAll(io.LimitReader(resp.Body, errBodyLimit)) + return nil, fmt.Errorf("openai_compat tts: endpoint error %d: %s", resp.StatusCode, strings.TrimSpace(string(errBody))) + } + + audioBytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("openai_compat tts: read response: %w", err) + } + + ext, mime := formatToExtMime(format) + return &audio.SynthResult{ + Audio: audioBytes, + Extension: ext, + MimeType: mime, + }, nil +} + +// formatToExtMime maps an OpenAI response_format to a file extension and MIME +// type. Unknown formats are passed through as their own extension with a +// generic MIME type rather than guessed at. +func formatToExtMime(format string) (ext, mime string) { + switch format { + case "mp3": + return "mp3", "audio/mpeg" + case "opus": + return "ogg", "audio/ogg" + case "aac": + return "aac", "audio/aac" + case "flac": + return "flac", "audio/flac" + case "wav": + return "wav", "audio/wav" + case "pcm": + return "pcm", "audio/pcm" + default: + return format, "application/octet-stream" + } +} + +// firstNonEmpty returns the first non-empty string, or "" if all are empty. +func firstNonEmpty(values ...string) string { + for _, v := range values { + if v != "" { + return v + } + } + return "" +} + +// paramString reads a string param, returning "" when absent or not a string. +func paramString(params map[string]any, key string) string { + if params == nil { + return "" + } + v, ok := audio.GetNested(params, key) + if !ok { + return "" + } + s, ok := v.(string) + if !ok { + return "" + } + return s +} + +// paramFloat reads a numeric param, returning 0 when absent or not numeric. +func paramFloat(params map[string]any, key string) float64 { + if params == nil { + return 0 + } + v, ok := audio.GetNested(params, key) + if !ok { + return 0 + } + switch n := v.(type) { + case float64: + return n + case float32: + return float64(n) + case int: + return float64(n) + case int64: + return float64(n) + default: + return 0 + } +} diff --git a/internal/audio/openaicompat/tts_test.go b/internal/audio/openaicompat/tts_test.go new file mode 100644 index 00000000..3cc767f1 --- /dev/null +++ b/internal/audio/openaicompat/tts_test.go @@ -0,0 +1,209 @@ +package openaicompat_test + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/nextlevelbuilder/goclaw/internal/audio" + "github.com/nextlevelbuilder/goclaw/internal/audio/openaicompat" +) + +// piperVoice is a real Piper voice ID. It is deliberately used throughout: +// it is exactly the kind of self-hosted voice that the "openai" provider name +// would silently rewrite to "alloy". +const piperVoice = "fr_FR-gilles-low" + +func TestNewTTSProviderRequiresAPIBase(t *testing.T) { + _, err := openaicompat.NewTTSProvider(openaicompat.Config{}) + require.ErrorIs(t, err, openaicompat.ErrAPIBaseRequired) + + _, err = openaicompat.NewTTSProvider(openaicompat.Config{APIBase: " "}) + require.ErrorIs(t, err, openaicompat.ErrAPIBaseRequired, "whitespace-only api_base must not pass") +} + +func TestTTSProviderName(t *testing.T) { + provider, err := openaicompat.NewTTSProvider(openaicompat.Config{APIBase: "http://x/v1"}) + require.NoError(t, err) + assert.Equal(t, "openai_compat", provider.Name()) +} + +// TestTTSProviderNameEscapesOpenAIVoiceValidation is the regression guard for +// the reason this package exists. Named "openai", a self-hosted voice ID is +// replaced by "alloy" before the request is ever sent. +func TestTTSProviderNameEscapesOpenAIVoiceValidation(t *testing.T) { + provider, err := openaicompat.NewTTSProvider(openaicompat.Config{APIBase: "http://x/v1"}) + require.NoError(t, err) + + assert.True(t, audio.IsVoiceCompatible(provider.Name(), piperVoice), + "a self-hosted voice must survive validation under this provider name") + + filtered, changed := audio.FilterVoiceForProvider(provider.Name(), piperVoice, false) + assert.Equal(t, piperVoice, filtered) + assert.False(t, changed) + + // Contrast: the same voice under the "openai" name is silently replaced. + clobbered, changed := audio.FilterVoiceForProvider("openai", piperVoice, false) + assert.Equal(t, "alloy", clobbered) + assert.True(t, changed) +} + +func TestTTSSynthesize(t *testing.T) { + var gotBody map[string]any + var gotAuth, gotPath string + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotAuth = r.Header.Get("Authorization") + require.NoError(t, json.NewDecoder(r.Body).Decode(&gotBody)) + w.Header().Set("Content-Type", "audio/wav") + _, _ = w.Write([]byte("RIFFfake")) + })) + t.Cleanup(func() { srv.Close() }) + + provider, err := openaicompat.NewTTSProvider(openaicompat.Config{ + APIBase: srv.URL + "/v1", + APIKey: "secret", + TTSFormat: "wav", + TTSVoice: piperVoice, + }) + require.NoError(t, err) + + result, err := provider.Synthesize(context.Background(), "bonjour", audio.TTSOptions{}) + require.NoError(t, err) + + assert.Equal(t, "/v1/audio/speech", gotPath) + assert.Equal(t, "Bearer secret", gotAuth) + assert.Equal(t, "bonjour", gotBody["input"]) + assert.Equal(t, piperVoice, gotBody["voice"], "configured voice must reach the endpoint verbatim") + assert.Equal(t, "wav", gotBody["response_format"]) + assert.NotContains(t, gotBody, "model", "an empty model must be omitted, not sent blank") + + assert.Equal(t, []byte("RIFFfake"), result.Audio) + assert.Equal(t, "wav", result.Extension) + assert.Equal(t, "audio/wav", result.MimeType) +} + +func TestTTSSynthesizeResolutionOrder(t *testing.T) { + var gotBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.NoError(t, json.NewDecoder(r.Body).Decode(&gotBody)) + _, _ = w.Write([]byte("x")) + })) + t.Cleanup(func() { srv.Close() }) + + provider, err := openaicompat.NewTTSProvider(openaicompat.Config{ + APIBase: srv.URL, + TTSVoice: "config-voice", + TTSModel: "config-model", + TTSFormat: "wav", + }) + require.NoError(t, err) + + _, err = provider.Synthesize(context.Background(), "hi", audio.TTSOptions{ + Voice: "opts-voice", + Model: "opts-model", + Format: "pcm", + Params: map[string]any{"speed": 1.5}, + }) + require.NoError(t, err) + + assert.Equal(t, "opts-voice", gotBody["voice"], "per-request opts win over config") + assert.Equal(t, "opts-model", gotBody["model"]) + assert.Equal(t, "pcm", gotBody["response_format"]) + assert.InEpsilon(t, 1.5, gotBody["speed"], 0.0001) +} + +func TestTTSSynthesizeParamsFormatOverridesConfig(t *testing.T) { + var gotBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.NoError(t, json.NewDecoder(r.Body).Decode(&gotBody)) + _, _ = w.Write([]byte("x")) + })) + t.Cleanup(func() { srv.Close() }) + + provider, err := openaicompat.NewTTSProvider(openaicompat.Config{ + APIBase: srv.URL, + TTSFormat: "wav", + }) + require.NoError(t, err) + + _, err = provider.Synthesize(context.Background(), "hi", audio.TTSOptions{ + Params: map[string]any{"response_format": "mp3"}, + }) + require.NoError(t, err) + assert.Equal(t, "mp3", gotBody["response_format"]) +} + +func TestTTSSynthesizeOmitsAuthWhenNoKey(t *testing.T) { + var hasAuth bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, hasAuth = r.Header["Authorization"] + _, _ = w.Write([]byte("x")) + })) + t.Cleanup(func() { srv.Close() }) + + provider, err := openaicompat.NewTTSProvider(openaicompat.Config{APIBase: srv.URL}) + require.NoError(t, err) + + _, err = provider.Synthesize(context.Background(), "hi", audio.TTSOptions{}) + require.NoError(t, err) + assert.False(t, hasAuth, "self-hosted endpoints without auth must not receive an empty Bearer header") +} + +func TestTTSSynthesizeErrorStatus(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusBadRequest) + _, _ = io.WriteString(w, `{"error":"response_format must be wav or pcm"}`) + })) + t.Cleanup(func() { srv.Close() }) + + provider, err := openaicompat.NewTTSProvider(openaicompat.Config{APIBase: srv.URL}) + require.NoError(t, err) + + _, err = provider.Synthesize(context.Background(), "hi", audio.TTSOptions{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "400") + assert.Contains(t, err.Error(), "response_format must be wav or pcm", + "the endpoint's own message must survive so a format mismatch is diagnosable") +} + +func TestTTSFormatExtensionMapping(t *testing.T) { + tests := []struct { + format string + wantExt string + wantMIME string + }{ + {"mp3", "mp3", "audio/mpeg"}, + {"opus", "ogg", "audio/ogg"}, + {"wav", "wav", "audio/wav"}, + {"pcm", "pcm", "audio/pcm"}, + {"flac", "flac", "audio/flac"}, + {"aac", "aac", "audio/aac"}, + } + for _, tc := range tests { + t.Run(tc.format, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("x")) + })) + t.Cleanup(func() { srv.Close() }) + + provider, err := openaicompat.NewTTSProvider(openaicompat.Config{ + APIBase: srv.URL, + TTSFormat: tc.format, + }) + require.NoError(t, err) + + result, err := provider.Synthesize(context.Background(), "hi", audio.TTSOptions{}) + require.NoError(t, err) + assert.Equal(t, tc.wantExt, result.Extension) + assert.Equal(t, tc.wantMIME, result.MimeType) + }) + } +} diff --git a/internal/config/config_channels.go b/internal/config/config_channels.go index b3855436..c87ee3d4 100644 --- a/internal/config/config_channels.go +++ b/internal/config/config_channels.go @@ -576,16 +576,33 @@ type SessionsConfig struct { // TtsConfig configures text-to-speech. // Matching TS src/config/types.tts.ts. type TtsConfig struct { - Provider string `json:"provider,omitempty"` // "openai", "elevenlabs", "edge", "minimax", "gemini" - Auto string `json:"auto,omitempty"` // "off" (default), "always", "inbound", "tagged" - Mode string `json:"mode,omitempty"` // "final" (default), "all" - MaxLength int `json:"max_length,omitempty"` // max text length before truncation (default 1500) - TimeoutMs int `json:"timeout_ms,omitempty"` // API timeout in ms (default 30000) - OpenAI TtsOpenAIConfig `json:"openai"` - ElevenLabs TtsElevenLabsConfig `json:"elevenlabs"` - Edge TtsEdgeConfig `json:"edge"` - MiniMax TtsMiniMaxConfig `json:"minimax"` - Gemini TtsGeminiConfig `json:"gemini"` + Provider string `json:"provider,omitempty"` // "openai", "openai_compat", "elevenlabs", "edge", "minimax", "gemini" + Auto string `json:"auto,omitempty"` // "off" (default), "always", "inbound", "tagged" + Mode string `json:"mode,omitempty"` // "final" (default), "all" + MaxLength int `json:"max_length,omitempty"` // max text length before truncation (default 1500) + TimeoutMs int `json:"timeout_ms,omitempty"` // API timeout in ms (default 30000) + OpenAI TtsOpenAIConfig `json:"openai"` + OpenAICompat TtsOpenAICompatConfig `json:"openai_compat"` + ElevenLabs TtsElevenLabsConfig `json:"elevenlabs"` + Edge TtsEdgeConfig `json:"edge"` + MiniMax TtsMiniMaxConfig `json:"minimax"` + Gemini TtsGeminiConfig `json:"gemini"` +} + +// TtsOpenAICompatConfig configures a self-hosted OpenAI-compatible audio +// endpoint (gpu-manager, Speaches, vLLM, LocalAI, llama.cpp, Ollama) for both +// TTS and STT. Kept separate from TtsOpenAIConfig because the voice namespace +// is the engine's own — see internal/audio/openaicompat for why the provider +// name must not be "openai". +// +// Setting api_base is what enables the provider; there is no vendor default. +type TtsOpenAICompatConfig struct { + APIBase string `json:"api_base,omitempty"` // required to enable, e.g. "http://gpu-manager:8080/v1" + APIKey string `json:"api_key,omitempty"` // optional; omitted entirely when empty + Model string `json:"model,omitempty"` // TTS model; optional when the engine picks it from the voice + Voice string `json:"voice,omitempty"` // engine-specific voice ID, e.g. "fr_FR-gilles-low" + Format string `json:"format,omitempty"` // response_format; default "mp3", use "wav" for engines without an encoder + STTModel string `json:"stt_model,omitempty"` // transcription model; optional } // TtsGeminiConfig configures the Google Gemini TTS provider.