package tools import ( "context" "fmt" "io" "log/slog" "net" "os" "path/filepath" "strings" "time" "github.com/nextlevelbuilder/goclaw/internal/providers" "github.com/nextlevelbuilder/goclaw/internal/security" usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps" ) // --- Context helpers for media video --- const ctxMediaVideoRefs toolContextKey = "tool_media_video_refs" // WithMediaVideoRefs stores video MediaRefs in context for read_video tool access. func WithMediaVideoRefs(ctx context.Context, refs []providers.MediaRef) context.Context { return context.WithValue(ctx, ctxMediaVideoRefs, refs) } // MediaVideoRefsFromCtx retrieves stored video MediaRefs from context. func MediaVideoRefsFromCtx(ctx context.Context) []providers.MediaRef { v, _ := ctx.Value(ctxMediaVideoRefs).([]providers.MediaRef) return v } // --- ReadVideoTool --- // videoMaxBytes is the max file size for video analysis (100MB). const videoMaxBytes = 100 * 1024 * 1024 const videoURLPinnedIPParam = "_pinned_ip" // videoLocalPathParam forwards the resolved local file path so the budget probe can read the real file. const videoLocalPathParam = "_local_path" // videoProviderPriority is the order in which providers are tried for video analysis. // OpenAI excluded — no native video upload in chat completions. var videoProviderPriority = []string{"gemini", "openrouter"} // videoModelDefaults maps provider names to preferred video-capable models. var videoModelDefaults = map[string]string{ "gemini": "gemini-2.5-flash", "openrouter": "google/gemini-2.5-flash", } // ReadVideoTool uses a video-capable provider to analyze video files // attached to the current conversation. type ReadVideoTool struct { registry *providers.Registry mediaLoader MediaPathLoader usageCaps *usagecaps.Service // streamToGemini is injected per instance rather than through a package // global, so parallel tests cannot race each other over one seam. streamToGemini geminiStreamFn } // geminiStreamFn is the upload a gate-passing URL is handed to. type geminiStreamFn func(ctx context.Context, apiKey, model, prompt string, reader io.Reader, contentLength int64, mime string, httpTimeout time.Duration) (*providers.ChatResponse, error) func NewReadVideoTool(registry *providers.Registry, mediaLoader MediaPathLoader) *ReadVideoTool { return &ReadVideoTool{registry: registry, mediaLoader: mediaLoader, streamToGemini: geminiFileAPICallStream} } // setStreamToGeminiForTest observes what reaches the upload without contacting a provider. func (t *ReadVideoTool) setStreamToGeminiForTest(fn geminiStreamFn) { t.streamToGemini = fn } func (t *ReadVideoTool) SetUsageCapService(svc *usagecaps.Service) { t.usageCaps = svc } func (t *ReadVideoTool) Name() string { return "read_video" } func (t *ReadVideoTool) Description() string { return "Analyze video files attached to the conversation. " + "Use when you see tags and need to describe, summarize, or analyze video content. " + "A workspace-relative path such as inputs/clip.mp4 may also be provided. Specify what you want to extract or analyze." } func (t *ReadVideoTool) Parameters() map[string]any { return map[string]any{ "type": "object", "properties": map[string]any{ "prompt": map[string]any{ "type": "string", "description": "What to analyze. E.g. 'Describe what happens in this video', 'Summarize the key scenes', 'What text appears on screen?'", }, "media_id": map[string]any{ "type": "string", "description": "Optional: specific media_id from tag. If omitted, uses most recent video.", }, "url": map[string]any{ "type": "string", "description": "Optional URL to a video file. Use this to analyze videos hosted online.", }, "path": map[string]any{ "type": "string", "description": "Optional workspace-relative video path. Delegated inputs use inputs/.", }, }, "required": []string{"prompt"}, } } func (t *ReadVideoTool) Execute(ctx context.Context, args map[string]any) *Result { prompt, _ := args["prompt"].(string) if prompt == "" { prompt = "Analyze this video and describe its contents." } mediaID, _ := args["media_id"].(string) videoURL, _ := args["url"].(string) videoArg, _ := args["path"].(string) sourceCount := 0 for _, source := range []string{mediaID, videoURL, videoArg} { if source != "" { sourceCount++ } } if sourceCount > 1 { if videoArg == "" && mediaID != "" && videoURL != "" { return ErrorResult("Both 'media_id' and 'url' parameters cannot be specified. Choose only one.") } return ErrorResult("Only one of 'media_id', 'url', or 'path' may be specified.") } var data []byte var videoMime string var videoPath string var pinnedIP net.IP if videoURL != "" { validatedURL, validatedIP, err := security.Validate(videoURL) if err != nil { return ErrorResult(fmt.Sprintf("Invalid video URL: %v", err)) } pinnedIP = validatedIP // Infer MIME type from URL extension ext := filepath.Ext(validatedURL.Path) videoMime = mimeFromVideoExt(ext) } else { var mime string var err error if videoArg != "" { videoPath, err = resolveStructuredMediaPath(ctx, videoArg, "video") mime = mimeFromVideoExt(filepath.Ext(videoPath)) } else { videoPath, mime, err = t.resolveVideoFile(ctx, mediaID) } if err != nil { return ErrorResult(err.Error()) } videoMime = mime slog.Info("read_video: resolved file", "mime", videoMime, "media_id", mediaID, "logical_path", videoArg) fileData, err := os.ReadFile(videoPath) if err != nil { if videoArg != "" && IsDelegationArtifactRun(ctx) { return ErrorResult("Failed to read delegation video input") } return ErrorResult(fmt.Sprintf("Failed to read video file: %v", err)) } slog.Info("read_video: file loaded", "size_bytes", len(fileData)) if len(fileData) > videoMaxBytes { return ErrorResult(fmt.Sprintf("Video too large: %d bytes (max %d)", len(fileData), videoMaxBytes)) } data = fileData } chain := ResolveMediaProviderChain(ctx, "read_video", "", "", videoProviderPriority, videoModelDefaults, t.registry) for i := range chain { if chain[i].Params == nil { chain[i].Params = make(map[string]any) } chain[i].Params["prompt"] = prompt chain[i].Params["data"] = data chain[i].Params["url"] = videoURL chain[i].Params["mime"] = videoMime chain[i].Params[videoLocalPathParam] = videoPath if pinnedIP != nil { chain[i].Params[videoURLPinnedIPParam] = pinnedIP } } chainResult, err := ExecuteWithChain(ctx, chain, t.registry, t.callProvider) if err != nil { return ErrorResult(fmt.Sprintf("Video analysis failed: %v", err)) } result := NewResult(string(chainResult.Data)) result.Usage = chainResult.Usage result.Provider = chainResult.Provider result.Model = chainResult.Model return result } // mimeFromVideoExt returns MIME type for video file extensions. func mimeFromVideoExt(ext string) string { switch strings.ToLower(ext) { case ".mp4": return "video/mp4" case ".webm": return "video/webm" case ".mov": return "video/quicktime" case ".avi": return "video/x-msvideo" case ".mkv": return "video/x-matroska" case ".wmv": return "video/x-ms-wmv" case ".flv": return "video/x-flv" case ".3gp": return "video/3gpp" case ".mpeg", ".mpg": return "video/mpeg" default: return "video/mp4" } }