feat(tools): add read_audio, read_video, create_video tools and fix system prompt tool filtering

- Add read_audio tool with Gemini File API, OpenAI input_audio, and fallback support
- Add read_video tool with Gemini File API and base64 fallback for video analysis
- Add create_video tool with Gemini Veo and OpenRouter chat completions support
- Add shared gemini_file_api.go for upload → poll → generateContent pipeline
- Add shared openai_compat_call.go for custom JSON chat completions
- Fix system prompt showing denied tools: use filteredToolNames() instead of tools.List()
- Wire audio/video MediaRef context propagation in agent loop
- Register new tools in seed data, policy groups, and web UI settings
- Enforce duration (max 30s) and aspect_ratio limits on create_video
This commit is contained in:
viettranx committed 2026-03-08 14:43:18 +07:00
1 parent ea185b3f6c
commit 691ddce8fb
15 files changed
+1484 -7

No files matched your search

+12
View File
@@ -51,6 +51,18 @@ func builtinToolSeedData() []store.BuiltinToolDef {
Settings: json.RawMessage(`{"provider":"openrouter","model":"google/gemini-2.5-flash-image"}`),
Requires: []string{"image_gen_provider"},
},
{Name: "read_audio", DisplayName: "Read Audio", Description: "Analyze audio files (speech, music, sounds) using an audio-capable LLM provider", Category: "media", Enabled: true,
Settings: json.RawMessage(`{"provider":"gemini","model":"gemini-2.5-flash"}`),
Requires: []string{"audio_provider"},
},
{Name: "read_video", DisplayName: "Read Video", Description: "Analyze video files using a video-capable LLM provider", Category: "media", Enabled: true,
Settings: json.RawMessage(`{"provider":"gemini","model":"gemini-2.5-flash"}`),
Requires: []string{"video_provider"},
},
{Name: "create_video", DisplayName: "Create Video", Description: "Generate videos from text descriptions using AI", Category: "media", Enabled: true,
Settings: json.RawMessage(`{"provider":"gemini","model":"veo-3.0-generate-preview"}`),
Requires: []string{"video_gen_provider"},
},
{Name: "tts", DisplayName: "Text to Speech", Description: "Convert text to speech audio", Category: "media", Enabled: true,
Requires: []string{"tts_provider"},
Metadata: json.RawMessage(`{"config_hint":"Config → TTS"}`),
+5 -2
View File
@@ -77,9 +77,12 @@ func wireExtras(
_ = mediaStore.DeleteSession(sessionKey)
}
}
// Register document analysis tool (needs mediaStore for file access).
// Register media analysis tools (need mediaStore for file access).
toolsReg.Register(tools.NewReadDocumentTool(providerReg, mediaStore))
slog.Info("read_document tool registered")
toolsReg.Register(tools.NewReadAudioTool(providerReg, mediaStore))
toolsReg.Register(tools.NewReadVideoTool(providerReg, mediaStore))
toolsReg.Register(tools.NewCreateVideoTool(providerReg))
slog.Info("media tools registered", "tools", "read_document,read_audio,read_video,create_video")
}
// 2. User seeding callback: seeds per-user context files on first chat