diff --git a/internal/agent/systemprompt.go b/internal/agent/systemprompt.go index 6ae0fcf2..c88f164a 100644 --- a/internal/agent/systemprompt.go +++ b/internal/agent/systemprompt.go @@ -206,9 +206,9 @@ var coreToolSummaries = map[string]string{ "session_status": "Show session status (model, tokens, compaction count)", "sessions_history": "Fetch message history for a session", "sessions_send": "Send a message into another session", - "read_image": "Analyze images — call with path from tags", + "read_image": "Analyze images — call with path from tags, or a direct HTTP/HTTPS URL via the 'url' parameter", "read_audio": "Analyze audio — call with media_id from tags", - "read_video": "Analyze video — call with media_id from tags", + "read_video": "Analyze video — call with media_id from tags, or a direct HTTP/HTTPS URL via the 'url' parameter", "create_video": "Generate videos from text descriptions using AI", "read_document": "Analyze documents (PDF, DOCX) from tags. If fails, use a skill instead. Path is directly accessible", "create_image": "Generate images from text descriptions using AI", diff --git a/internal/providers/anthropic_request.go b/internal/providers/anthropic_request.go index 306eb158..76aaae84 100644 --- a/internal/providers/anthropic_request.go +++ b/internal/providers/anthropic_request.go @@ -98,7 +98,7 @@ func (p *AnthropicProvider) buildRequestBody(model string, req ChatRequest, stre systemBlocks = append(systemBlocks, splitSystemPromptForCache(msg.Content)...) case "user": - if len(msg.Images) > 0 { + if len(msg.Images) > 0 || len(msg.Videos) > 0 { var blocks []map[string]any for _, img := range msg.Images { blocks = append(blocks, map[string]any{ @@ -110,6 +110,7 @@ func (p *AnthropicProvider) buildRequestBody(model string, req ChatRequest, stre }, }) } + // Videos are not supported by Anthropic, they are omitted here. if msg.Content != "" { blocks = append(blocks, map[string]any{ "type": "text", diff --git a/internal/providers/codex_build.go b/internal/providers/codex_build.go index 5a9a941a..802f3ecc 100644 --- a/internal/providers/codex_build.go +++ b/internal/providers/codex_build.go @@ -34,7 +34,7 @@ func (p *CodexProvider) buildRequestBody(req ChatRequest, stream bool) map[strin } case "user": - if len(m.Images) > 0 { + if len(m.Images) > 0 || len(m.Videos) > 0 { var parts []map[string]any for _, img := range m.Images { parts = append(parts, map[string]any{ @@ -42,6 +42,7 @@ func (p *CodexProvider) buildRequestBody(req ChatRequest, stream bool) map[strin "image_url": fmt.Sprintf("data:%s;base64,%s", img.MimeType, img.Data), }) } + // Videos are not supported by Codex, they are omitted here. if m.Content != "" { parts = append(parts, map[string]any{ "type": "input_text", diff --git a/internal/providers/openai_request.go b/internal/providers/openai_request.go index 401d6dcc..fb96b470 100644 --- a/internal/providers/openai_request.go +++ b/internal/providers/openai_request.go @@ -64,9 +64,9 @@ func (p *OpenAIProvider) buildRequestBody(model string, req ChatRequest, stream // Include content; omit empty content for assistant messages with tool_calls // (Gemini rejects empty content → "must include at least one parts field"). - if m.Role == "user" && len(m.Images) > 0 { + if m.Role == "user" && (len(m.Images) > 0 || len(m.Videos) > 0) { var parts []map[string]any - // Text before images — Together / Qwen vision examples use this order; OpenAI accepts both. + // Text before images/videos — Together / Qwen vision examples use this order; OpenAI accepts both. if m.Content != "" { parts = append(parts, map[string]any{ "type": "text", @@ -74,10 +74,26 @@ func (p *OpenAIProvider) buildRequestBody(model string, req ChatRequest, stream }) } for _, img := range m.Images { + urlVal := img.URL + if urlVal == "" { + urlVal = fmt.Sprintf("data:%s;base64,%s", img.MimeType, img.Data) + } parts = append(parts, map[string]any{ "type": "image_url", "image_url": map[string]any{ - "url": fmt.Sprintf("data:%s;base64,%s", img.MimeType, img.Data), + "url": urlVal, + }, + }) + } + for _, vid := range m.Videos { + urlVal := vid.URL + if urlVal == "" { + urlVal = fmt.Sprintf("data:%s;base64,%s", vid.MimeType, vid.Data) + } + parts = append(parts, map[string]any{ + "type": "video_url", + "video_url": map[string]any{ + "url": urlVal, }, }) } diff --git a/internal/providers/openai_test.go b/internal/providers/openai_test.go index ccf6528d..8f04bc85 100644 --- a/internal/providers/openai_test.go +++ b/internal/providers/openai_test.go @@ -240,6 +240,60 @@ func TestBuildRequestBody_MultimodalTextBeforeImages(t *testing.T) { } } +func TestBuildRequestBody_MultimodalWithImageURL(t *testing.T) { + p := NewOpenAIProvider("test", "key", "https://api.openai.com/v1", "gpt-4") + req := ChatRequest{ + Messages: []Message{ + { + Role: "user", + Content: "describe", + Images: []ImageContent{ + {URL: "https://example.com/image.png"}, + }, + }, + }, + } + body := p.buildRequestBody("gpt-4o", req, false) + msgs := body["messages"].([]map[string]any) + parts, ok := msgs[0]["content"].([]map[string]any) + if !ok || len(parts) < 2 { + t.Fatalf("want multimodal parts, got %v", msgs[0]["content"]) + } + imgPart := parts[1]["image_url"].(map[string]any) + if urlVal, _ := imgPart["url"].(string); urlVal != "https://example.com/image.png" { + t.Errorf("expected URL to be https://example.com/image.png, got %q", urlVal) + } +} + +func TestBuildRequestBody_MultimodalWithVideoURL(t *testing.T) { + p := NewOpenAIProvider("test", "key", "https://api.openai.com/v1", "gpt-4") + req := ChatRequest{ + Messages: []Message{ + { + Role: "user", + Content: "describe video", + Videos: []VideoContent{ + {MimeType: "video/mp4", URL: "https://example.com/video.mp4"}, + }, + }, + }, + } + body := p.buildRequestBody("gpt-4o", req, false) + msgs := body["messages"].([]map[string]any) + parts, ok := msgs[0]["content"].([]map[string]any) + if !ok || len(parts) < 2 { + t.Fatalf("want multimodal parts, got %v", msgs[0]["content"]) + } + videoPart, ok := parts[1]["video_url"].(map[string]any) + if !ok { + t.Fatalf("expected video_url part, got %v", parts[1]) + } + if urlVal, _ := videoPart["url"].(string); urlVal != "https://example.com/video.mp4" { + t.Errorf("expected URL to be https://example.com/video.mp4, got %q", urlVal) + } +} + + func TestBuildRequestBody_TogetherDetectedByProviderType(t *testing.T) { // Together behind reverse proxy — detected by providerType, not URL. p := NewOpenAIProvider("my-proxy", "key", "https://proxy.internal/v1", "") diff --git a/internal/providers/types.go b/internal/providers/types.go index d7f03413..900eeceb 100644 --- a/internal/providers/types.go +++ b/internal/providers/types.go @@ -113,13 +113,22 @@ type StreamChunk struct { Images []ImageContent `json:"images,omitempty"` // image generation frames (Codex) } -// ImageContent represents a base64-encoded image for vision-capable models. +// ImageContent represents an image (either base64-encoded or a direct URL) for vision-capable models. type ImageContent struct { MimeType string `json:"mime_type"` // e.g. "image/jpeg" Data string `json:"data"` // base64-encoded image bytes + URL string `json:"url,omitempty"` // URL of the image Partial bool `json:"partial,omitempty"` // true for intermediate frames (Codex image_generation_call) } +// VideoContent represents a video (either base64-encoded or a direct URL) for video-capable models. +type VideoContent struct { + MimeType string `json:"mime_type"` // e.g. "video/mp4" + Data string `json:"data"` // base64-encoded video bytes + URL string `json:"url,omitempty"` // URL of the video + Partial bool `json:"partial,omitempty"` // true for intermediate frames +} + // MediaRef is a lightweight reference to a persistently stored media file. // Stored in session JSONB (~60 bytes each) instead of megabytes for base64. // On reload, MediaRefs are resolved to file paths and loaded into Images (for images). @@ -137,6 +146,7 @@ type Message struct { Content string `json:"content"` Thinking string `json:"thinking,omitempty"` // reasoning_content for thinking models (Kimi, DeepSeek, etc.) Images []ImageContent `json:"-"` // vision: base64 images (runtime only, never persisted to DB) + Videos []VideoContent `json:"-"` // vision: base64 videos (runtime only, never persisted to DB) MediaRefs []MediaRef `json:"media_refs,omitempty"` // persistent media file references ToolCalls []ToolCall `json:"tool_calls,omitempty"` ToolCallID string `json:"tool_call_id,omitempty"` // for role="tool" responses diff --git a/internal/tools/gemini_file_api.go b/internal/tools/gemini_file_api.go index fe96b941..4ab1dd64 100644 --- a/internal/tools/gemini_file_api.go +++ b/internal/tools/gemini_file_api.go @@ -264,3 +264,153 @@ func parseGeminiResponse(respBody []byte) (*providers.ChatResponse, error) { }, }, nil } + +// geminiFileUploadStream uploads a file stream to Gemini File API using resumable upload protocol. +// Returns the file name (e.g. "files/abc123") and file URI for use in generateContent. +func geminiFileUploadStream(ctx context.Context, apiKey, displayName string, reader io.Reader, contentLength int64, mime string) (fileName, fileURI string, err error) { + // Step 1: Initiate resumable upload. + initBody, _ := json.Marshal(map[string]any{ + "file": map[string]string{"display_name": displayName}, + }) + initReq, err := http.NewRequestWithContext(ctx, "POST", geminiUploadBase+"?key="+apiKey, bytes.NewReader(initBody)) + if err != nil { + return "", "", fmt.Errorf("create init request: %w", err) + } + initReq.Header.Set("Content-Type", "application/json") + initReq.Header.Set("X-Goog-Upload-Protocol", "resumable") + initReq.Header.Set("X-Goog-Upload-Command", "start") + initReq.Header.Set("X-Goog-Upload-Header-Content-Length", fmt.Sprintf("%d", contentLength)) + initReq.Header.Set("X-Goog-Upload-Header-Content-Type", mime) + + client := &http.Client{Timeout: 60 * time.Second} + initResp, err := client.Do(initReq) + if err != nil { + return "", "", fmt.Errorf("init upload: %w", err) + } + defer initResp.Body.Close() + io.ReadAll(initResp.Body) // drain + + if initResp.StatusCode != 200 { + return "", "", fmt.Errorf("init upload HTTP %d", initResp.StatusCode) + } + + uploadURL := initResp.Header.Get("X-Goog-Upload-URL") + if uploadURL == "" { + return "", "", fmt.Errorf("no upload URL in response headers") + } + + // Step 2: Upload file stream. + uploadReq, err := http.NewRequestWithContext(ctx, "POST", uploadURL, reader) + if err != nil { + return "", "", fmt.Errorf("create upload request: %w", err) + } + uploadReq.ContentLength = contentLength + uploadReq.Header.Set("Content-Length", fmt.Sprintf("%d", contentLength)) + uploadReq.Header.Set("X-Goog-Upload-Offset", "0") + uploadReq.Header.Set("X-Goog-Upload-Command", "upload, finalize") + + // Do not set global Timeout on the HTTP client here because we are piping a potentially large stream. + uploadClient := &http.Client{} + uploadResp, err := uploadClient.Do(uploadReq) + if err != nil { + return "", "", fmt.Errorf("upload stream: %w", err) + } + defer uploadResp.Body.Close() + + respBody, err := io.ReadAll(uploadResp.Body) + if err != nil { + return "", "", fmt.Errorf("read upload response: %w", err) + } + if uploadResp.StatusCode != 200 { + return "", "", fmt.Errorf("upload HTTP %d: %s", uploadResp.StatusCode, truncateStr(string(respBody), 500)) + } + + var uploadResult struct { + File struct { + Name string `json:"name"` + URI string `json:"uri"` + State string `json:"state"` + } `json:"file"` + } + if err := json.Unmarshal(respBody, &uploadResult); err != nil { + return "", "", fmt.Errorf("parse upload response: %w", err) + } + + // Only return URI if file is already ACTIVE; otherwise caller must poll. + if uploadResult.File.State == "ACTIVE" { + return uploadResult.File.Name, uploadResult.File.URI, nil + } + return uploadResult.File.Name, "", nil +} + +// geminiFileAPICallStream uploads a file stream via Gemini File API, polls until ready, +// then calls generateContent with file_data reference. +func geminiFileAPICallStream(ctx context.Context, apiKey, model, prompt string, reader io.Reader, contentLength int64, mime string, httpTimeout time.Duration) (*providers.ChatResponse, error) { + displayName := fmt.Sprintf("goclaw_%d", time.Now().UnixNano()) + + slog.Info("gemini file api: uploading stream", "size", contentLength, "mime", mime) + fileName, fileURI, err := geminiFileUploadStream(ctx, apiKey, displayName, reader, contentLength, mime) + if err != nil { + return nil, fmt.Errorf("upload: %w", err) + } + slog.Info("gemini file api: uploaded stream", "name", fileName) + + // If file URI not returned directly, poll for it. + if fileURI == "" { + slog.Info("gemini file api: polling for active state", "name", fileName) + fileURI, err = geminiFilePoll(ctx, apiKey, fileName) + if err != nil { + return nil, fmt.Errorf("poll: %w", err) + } + } + slog.Info("gemini file api: file active", "uri", fileURI) + + // Call generateContent with file_data reference. + body := map[string]any{ + "contents": []map[string]any{ + { + "parts": []map[string]any{ + {"file_data": map[string]any{"mime_type": mime, "file_uri": fileURI}}, + {"text": prompt}, + }, + }, + }, + "generationConfig": map[string]any{ + "maxOutputTokens": 16384, + "temperature": 0.2, + }, + } + + bodyJSON, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("marshal request: %w", err) + } + + url := fmt.Sprintf("https://generativelanguage.googleapis.com/v1beta/models/%s:generateContent?key=%s", model, apiKey) + httpReq, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(bodyJSON)) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + httpReq.Header.Set("Content-Type", "application/json") + + if httpTimeout == 0 { + httpTimeout = 120 * time.Second + } + client := &http.Client{Timeout: httpTimeout} + httpResp, err := client.Do(httpReq) + if err != nil { + return nil, fmt.Errorf("HTTP request: %w", err) + } + defer httpResp.Body.Close() + + respBody, err := io.ReadAll(httpResp.Body) + if err != nil { + return nil, fmt.Errorf("read response: %w", err) + } + if httpResp.StatusCode != 200 { + return nil, fmt.Errorf("HTTP %d: %s", httpResp.StatusCode, truncateStr(string(respBody), 500)) + } + + // Parse — same response format as geminiNativeDocumentCall. + return parseGeminiResponse(respBody) +} diff --git a/internal/tools/read_image.go b/internal/tools/read_image.go index 892a6dfb..8664184d 100644 --- a/internal/tools/read_image.go +++ b/internal/tools/read_image.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/nextlevelbuilder/goclaw/internal/providers" + "github.com/nextlevelbuilder/goclaw/internal/security" usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps" ) @@ -62,7 +63,7 @@ func (t *ReadImageTool) SetUsageCapService(svc *usagecaps.Service) { func (t *ReadImageTool) Name() string { return "read_image" } func (t *ReadImageTool) Description() string { - return "Analyze images using vision AI. Works with: (1) images sent by the user ( tags), (2) workspace/generated image files (pass a file path)." + return "Analyze images using vision AI. Works with images sent by the user, workspace/generated image files, or public HTTP/HTTPS image URLs." } func (t *ReadImageTool) Parameters() map[string]any { @@ -77,6 +78,10 @@ func (t *ReadImageTool) Parameters() map[string]any { "type": "string", "description": "Optional file path to an image in the workspace. Use this for generated images or attachments. If omitted, analyzes images from the conversation.", }, + "url": map[string]any{ + "type": "string", + "description": "Optional URL to an image. Use this to analyze images hosted online.", + }, }, "required": []string{"prompt"}, } @@ -91,18 +96,32 @@ func (t *ReadImageTool) Execute(ctx context.Context, args map[string]any) *Resul prompt = "Describe this image in detail." } + imgPath, _ := args["path"].(string) + imgURL, _ := args["url"].(string) + + if imgPath != "" && imgURL != "" { + return ErrorResult("Both 'path' and 'url' parameters cannot be specified. Choose only one.") + } + // If path is provided, load image from workspace file images := MediaImagesFromCtx(ctx) - if imgPath, _ := args["path"].(string); imgPath != "" { + if imgPath != "" { fileImages, err := t.loadImageFromPath(ctx, imgPath) if err != nil { return ErrorResult(err.Error()) } images = fileImages + } else if imgURL != "" { + if _, _, err := security.Validate(imgURL); err != nil { + return ErrorResult(fmt.Sprintf("Invalid image URL: %v", err)) + } + images = []providers.ImageContent{{ + URL: imgURL, + }} } if len(images) == 0 { - return ErrorResult("No images available. Either send an image in the chat or provide a file path with the 'path' parameter.") + return ErrorResult("No images available. Either send an image in the chat, provide a file path with 'path', or provide an image URL with 'url'.") } chain := ResolveMediaProviderChain(ctx, "read_image", "", "", @@ -138,6 +157,24 @@ func (t *ReadImageTool) callProvider(ctx context.Context, cp credentialProvider, prompt := GetParamString(params, "prompt", "Describe this image in detail.") images, _ := params["images"].([]providers.ImageContent) + for _, img := range images { + if img.URL == "" { + continue + } + if _, _, err := security.Validate(img.URL); err != nil { + return nil, nil, fmt.Errorf("invalid image URL: %w", err) + } + } + + // Anthropic Claude does not support URL references and requires base64-encoded image data. + if providerName == "anthropic" || providerName == "claude-cli" { + for _, img := range images { + if img.URL != "" && img.Data == "" { + return nil, nil, fmt.Errorf("provider %q does not support analyzing images directly from a URL", providerName) + } + } + } + // Get the full provider for Chat() access p, err := t.registry.Get(ctx, providerName) if err != nil { diff --git a/internal/tools/read_image_test.go b/internal/tools/read_image_test.go new file mode 100644 index 00000000..87ee177a --- /dev/null +++ b/internal/tools/read_image_test.go @@ -0,0 +1,71 @@ +package tools + +import ( + "context" + "strings" + "testing" + + "github.com/nextlevelbuilder/goclaw/internal/providers" +) + +func TestReadImage_BothPathAndUrl_Error(t *testing.T) { + tool := NewReadImageTool(nil) + + res := tool.Execute(context.Background(), map[string]any{ + "prompt": "describe this", + "path": "workspace/image.png", + "url": "https://example.com/image.png", + }) + + if !res.IsError { + t.Fatalf("expected error when both path and url are provided") + } + + if !strings.Contains(res.ForLLM, "Both 'path' and 'url' parameters cannot be specified") { + t.Errorf("unexpected error message: %s", res.ForLLM) + } +} + +func TestReadImage_PrivateURL_Error(t *testing.T) { + tool := NewReadImageTool(nil) + + res := tool.Execute(context.Background(), map[string]any{ + "prompt": "describe this", + "url": "http://127.0.0.1/image.png", + }) + + if !res.IsError { + t.Fatalf("expected error for private image URL") + } + if !strings.Contains(res.ForLLM, "Invalid image URL") { + t.Errorf("unexpected error message: %s", res.ForLLM) + } +} + +func TestReadImage_AnthropicURL_Error(t *testing.T) { + tool := NewReadImageTool(nil) + + params := map[string]any{ + "prompt": "describe this", + "images": []providers.ImageContent{ + { + URL: "https://93.184.216.34/image.png", + }, + }, + } + + _, _, err := tool.callProvider(context.Background(), nil, "anthropic", "claude-3-sonnet", params) + if err == nil { + t.Fatalf("expected error for anthropic provider with image URL") + } + + if !strings.Contains(err.Error(), "does not support analyzing images directly from a URL") { + t.Errorf("unexpected error message: %v", err) + } + + // Should also error for claude-cli + _, _, err = tool.callProvider(context.Background(), nil, "claude-cli", "claude-3-sonnet", params) + if err == nil { + t.Fatalf("expected error for claude-cli provider with image URL") + } +} diff --git a/internal/tools/read_video.go b/internal/tools/read_video.go index 46df93c4..48b1aaba 100644 --- a/internal/tools/read_video.go +++ b/internal/tools/read_video.go @@ -4,10 +4,13 @@ import ( "context" "fmt" "log/slog" + "net" "os" + "path/filepath" "strings" "github.com/nextlevelbuilder/goclaw/internal/providers" + "github.com/nextlevelbuilder/goclaw/internal/security" usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps" ) @@ -31,6 +34,8 @@ func MediaVideoRefsFromCtx(ctx context.Context) []providers.MediaRef { // videoMaxBytes is the max file size for video analysis (100MB). const videoMaxBytes = 100 * 1024 * 1024 +const videoURLPinnedIPParam = "_pinned_ip" + // videoProviderPriority is the order in which providers are tried for video analysis. // OpenAI excluded — no native video upload in chat completions. var videoProviderPriority = []string{"gemini", "openrouter"} @@ -77,6 +82,10 @@ func (t *ReadVideoTool) Parameters() map[string]any { "type": "string", "description": "Optional: specific media_id from tag. If omitted, uses most recent video.", }, + "url": map[string]any{ + "type": "string", + "description": "Optional URL to a video file. Use this to analyze videos hosted online.", + }, }, "required": []string{"prompt"}, } @@ -88,21 +97,43 @@ func (t *ReadVideoTool) Execute(ctx context.Context, args map[string]any) *Resul prompt = "Analyze this video and describe its contents." } mediaID, _ := args["media_id"].(string) + videoURL, _ := args["url"].(string) - videoPath, videoMime, err := t.resolveVideoFile(ctx, mediaID) - if err != nil { - return ErrorResult(err.Error()) + if mediaID != "" && videoURL != "" { + return ErrorResult("Both 'media_id' and 'url' parameters cannot be specified. Choose only one.") } - slog.Info("read_video: resolved file", "path", videoPath, "mime", videoMime, "media_id", mediaID) + var data []byte + var videoMime string + var pinnedIP net.IP - data, err := os.ReadFile(videoPath) - if err != nil { - return ErrorResult(fmt.Sprintf("Failed to read video file: %v", err)) - } - slog.Info("read_video: file loaded", "size_bytes", len(data)) - if len(data) > videoMaxBytes { - return ErrorResult(fmt.Sprintf("Video too large: %d bytes (max %d)", len(data), videoMaxBytes)) + if videoURL != "" { + validatedURL, validatedIP, err := security.Validate(videoURL) + if err != nil { + return ErrorResult(fmt.Sprintf("Invalid video URL: %v", err)) + } + pinnedIP = validatedIP + + // Infer MIME type from URL extension + ext := filepath.Ext(validatedURL.Path) + videoMime = mimeFromVideoExt(ext) + } else { + videoPath, mime, err := t.resolveVideoFile(ctx, mediaID) + if err != nil { + return ErrorResult(err.Error()) + } + videoMime = mime + slog.Info("read_video: resolved file", "path", videoPath, "mime", videoMime, "media_id", mediaID) + + fileData, err := os.ReadFile(videoPath) + if err != nil { + return ErrorResult(fmt.Sprintf("Failed to read video file: %v", err)) + } + slog.Info("read_video: file loaded", "size_bytes", len(fileData)) + if len(fileData) > videoMaxBytes { + return ErrorResult(fmt.Sprintf("Video too large: %d bytes (max %d)", len(fileData), videoMaxBytes)) + } + data = fileData } chain := ResolveMediaProviderChain(ctx, "read_video", "", "", @@ -114,7 +145,11 @@ func (t *ReadVideoTool) Execute(ctx context.Context, args map[string]any) *Resul } chain[i].Params["prompt"] = prompt chain[i].Params["data"] = data + chain[i].Params["url"] = videoURL chain[i].Params["mime"] = videoMime + if pinnedIP != nil { + chain[i].Params[videoURLPinnedIPParam] = pinnedIP + } } chainResult, err := ExecuteWithChain(ctx, chain, t.registry, t.callProvider) diff --git a/internal/tools/read_video_resolve.go b/internal/tools/read_video_resolve.go index f912fb96..646068cb 100644 --- a/internal/tools/read_video_resolve.go +++ b/internal/tools/read_video_resolve.go @@ -5,10 +5,14 @@ import ( "encoding/base64" "fmt" "log/slog" + "net" + "net/http" "path/filepath" + "strings" "time" "github.com/nextlevelbuilder/goclaw/internal/providers" + "github.com/nextlevelbuilder/goclaw/internal/security" ) // resolveVideoFile finds the video file path from context MediaRefs. @@ -60,16 +64,26 @@ func (t *ReadVideoTool) resolveVideoFile(ctx context.Context, mediaID string) (p // callProvider dispatches video analysis to the appropriate provider API. // Gemini: uses File API (upload → poll → file_data in generateContent). -// Others: falls back to base64 in image_url (OpenRouter routes to Gemini which handles video). +// Others: falls back to base64 or URL in video_url (OpenRouter routes to Gemini which handles video). func (t *ReadVideoTool) callProvider(ctx context.Context, cp credentialProvider, providerName, model string, params map[string]any) ([]byte, *providers.Usage, error) { prompt := GetParamString(params, "prompt", "Analyze this video and describe its contents.") data, _ := params["data"].([]byte) + videoURL, _ := params["url"].(string) mime := GetParamString(params, "mime", "video/mp4") + pinnedIP, _ := params[videoURLPinnedIPParam].(net.IP) + if videoURL != "" && pinnedIP == nil { + var err error + if _, pinnedIP, err = security.Validate(videoURL); err != nil { + return nil, nil, fmt.Errorf("invalid video URL: %w", err) + } + } // Gemini: use File API (requires credentials). ptype := GetParamString(params, "_provider_type", providerTypeFromName(providerName)) if cp != nil && ptype == "gemini" { - slog.Info("read_video: using gemini file API", "provider", providerName, "model", model, "size", len(data), "mime", mime) + var resp *providers.ChatResponse + var err error + chatReq := providers.ChatRequest{ Messages: []providers.Message{{Role: "user", Content: prompt}}, Model: model, @@ -79,7 +93,71 @@ func (t *ReadVideoTool) callProvider(ctx context.Context, cp credentialProvider, if reserveErr != nil { return nil, nil, reserveErr } - resp, err := geminiFileAPICall(ctx, cp.APIKey(), model, prompt, data, mime, 180*time.Second) + + if videoURL != "" { + slog.Info("read_video: streaming URL directly to Gemini File API", "provider", providerName, "model", model, "url", videoURL) + + // Send GET request to fetch the stream. + reqCtx := security.WithPinnedIP(ctx, pinnedIP) + req, getErr := http.NewRequestWithContext(reqCtx, "GET", videoURL, nil) + if getErr != nil { + if reservation != nil { + reservation.Reconcile(ctx, nil, getErr) + } + return nil, nil, fmt.Errorf("failed to create GET request for video URL: %w", getErr) + } + + // Use the shared SSRF-safe client so DNS stays pinned during streaming. + client := security.NewSafeClient(0) + httpResp, getErr := client.Do(req) + if getErr != nil { + if reservation != nil { + reservation.Reconcile(ctx, nil, getErr) + } + return nil, nil, fmt.Errorf("failed to fetch video URL: %w", getErr) + } + defer httpResp.Body.Close() + + if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 { + statusErr := fmt.Errorf("video URL returned status code %d", httpResp.StatusCode) + if reservation != nil { + reservation.Reconcile(ctx, nil, statusErr) + } + return nil, nil, statusErr + } + + // Validate Content-Length + contentLength := httpResp.ContentLength + if contentLength <= 0 { + invalidLenErr := fmt.Errorf("URL does not support static streaming (missing or invalid Content-Length: %d)", contentLength) + if reservation != nil { + reservation.Reconcile(ctx, nil, invalidLenErr) + } + return nil, nil, invalidLenErr + } + + // Check limits: maximum 2 GB + const videoMaxStreamBytes = 2 * 1024 * 1024 * 1024 // 2 GB + if contentLength > videoMaxStreamBytes { + limitErr := fmt.Errorf("video stream size (%d bytes) exceeds the maximum limit of 2 GB", contentLength) + if reservation != nil { + reservation.Reconcile(ctx, nil, limitErr) + } + return nil, nil, limitErr + } + + // Extract MIME type from Content-Type header if valid video type, otherwise use the inferred/passed one + contentType := httpResp.Header.Get("Content-Type") + if contentType != "" && strings.HasPrefix(contentType, "video/") { + mime = contentType + } + + resp, err = geminiFileAPICallStream(ctx, cp.APIKey(), model, prompt, httpResp.Body, contentLength, mime, 300*time.Second) + } else { + slog.Info("read_video: using gemini file API", "provider", providerName, "model", model, "size", len(data), "mime", mime) + resp, err = geminiFileAPICall(ctx, cp.APIKey(), model, prompt, data, mime, 180*time.Second) + } + if reservation != nil { reservation.Reconcile(ctx, resp, err) } @@ -89,19 +167,27 @@ func (t *ReadVideoTool) callProvider(ctx context.Context, cp credentialProvider, return []byte(resp.Content), resp.Usage, nil } - // Other providers: try standard Chat API with base64 as image_url (best effort). + // Other providers: try standard Chat API with base64 or URL as video_url (best effort). p, err := t.registry.Get(ctx, providerName) if err != nil { return nil, nil, fmt.Errorf("provider %q not available: %w", providerName, err) } - slog.Info("read_video: using chat API fallback", "provider", providerName, "model", model, "size", len(data)) + var vidContent providers.VideoContent + if videoURL != "" { + slog.Info("read_video: using chat API with direct video URL", "provider", providerName, "model", model, "url", videoURL) + vidContent = providers.VideoContent{MimeType: mime, URL: videoURL} + } else { + slog.Info("read_video: using chat API fallback with base64", "provider", providerName, "model", model, "size", len(data)) + vidContent = providers.VideoContent{MimeType: mime, Data: base64.StdEncoding.EncodeToString(data)} + } + chatReq := providers.ChatRequest{ Messages: []providers.Message{ { Role: "user", Content: prompt, - Images: []providers.ImageContent{{MimeType: mime, Data: base64.StdEncoding.EncodeToString(data)}}, + Videos: []providers.VideoContent{vidContent}, }, }, Model: model, diff --git a/internal/tools/read_video_test.go b/internal/tools/read_video_test.go new file mode 100644 index 00000000..3440f947 --- /dev/null +++ b/internal/tools/read_video_test.go @@ -0,0 +1,123 @@ +package tools + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/nextlevelbuilder/goclaw/internal/security" +) + +type mockCredentialProvider struct { + apiKey string + apiBase string +} + +func (m *mockCredentialProvider) APIKey() string { return m.apiKey } +func (m *mockCredentialProvider) APIBase() string { return m.apiBase } + +func TestReadVideo_BothMediaIdAndUrl_Error(t *testing.T) { + tool := NewReadVideoTool(nil, nil) + + res := tool.Execute(context.Background(), map[string]any{ + "prompt": "describe this video", + "media_id": "video-123", + "url": "https://example.com/video.mp4", + }) + + if !res.IsError { + t.Fatalf("expected error when both media_id and url are provided") + } + + if !strings.Contains(res.ForLLM, "Both 'media_id' and 'url' parameters cannot be specified") { + t.Errorf("unexpected error message: %s", res.ForLLM) + } +} + +func TestReadVideo_PrivateURL_Error(t *testing.T) { + tool := NewReadVideoTool(nil, nil) + + res := tool.Execute(context.Background(), map[string]any{ + "prompt": "describe this video", + "url": "http://127.0.0.1/video.mp4", + }) + + if !res.IsError { + t.Fatalf("expected error for private video URL") + } + if !strings.Contains(res.ForLLM, "Invalid video URL") { + t.Errorf("unexpected error message: %s", res.ForLLM) + } +} + +func TestReadVideo_GeminiURL_Validation(t *testing.T) { + security.SetAllowLoopbackForTest(true) + defer security.SetAllowLoopbackForTest(false) + + // Missing Content-Length should fail before upload. + ts1 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Transfer-Encoding", "chunked") + w.Write([]byte("chunked data mock video")) + })) + defer ts1.Close() + + tool := NewReadVideoTool(nil, nil) + cp := &mockCredentialProvider{apiKey: "test-key"} + + params1 := map[string]any{ + "prompt": "describe this video", + "url": ts1.URL, + "_provider_type": "gemini", + } + + _, _, err := tool.callProvider(context.Background(), cp, "gemini", "gemini-2.5-flash", params1) + if err == nil { + t.Fatalf("expected error for missing Content-Length") + } + if !strings.Contains(err.Error(), "URL does not support static streaming") { + t.Errorf("unexpected error for missing Content-Length: %v", err) + } + + // Content-Length over the Gemini File API limit should fail before upload. + ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", "2147483649") // 2GB + 1 byte + w.WriteHeader(http.StatusOK) + })) + defer ts2.Close() + + params2 := map[string]any{ + "prompt": "describe this video", + "url": ts2.URL, + "_provider_type": "gemini", + } + + _, _, err = tool.callProvider(context.Background(), cp, "gemini", "gemini-2.5-flash", params2) + if err == nil { + t.Fatalf("expected error for Content-Length exceeding 2GB") + } + if !strings.Contains(err.Error(), "exceeds the maximum limit of 2 GB") { + t.Errorf("unexpected error for limit exceed: %v", err) + } + + // Non-2xx status should be reported before upload. + ts3 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer ts3.Close() + + params3 := map[string]any{ + "prompt": "describe this video", + "url": ts3.URL, + "_provider_type": "gemini", + } + + _, _, err = tool.callProvider(context.Background(), cp, "gemini", "gemini-2.5-flash", params3) + if err == nil { + t.Fatalf("expected error for HTTP 404 status code") + } + if !strings.Contains(err.Error(), "video URL returned status code 404") { + t.Errorf("unexpected error for HTTP status code: %v", err) + } +} diff --git a/pkg/browser/browser_remote.go b/pkg/browser/browser_remote.go index 68017bda..c3b5628a 100644 --- a/pkg/browser/browser_remote.go +++ b/pkg/browser/browser_remote.go @@ -254,7 +254,7 @@ func resolveRemoteCDP(remoteURL string) (string, error) { defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("/json/version returned HTTP %d", resp.StatusCode) + return "", fmt.Errorf("query /json/version at %s returned HTTP %d", versionURL, resp.StatusCode) } var ver struct {