From 0f9dd1f1bcb0fc47382e3d651c02357cc877b02a Mon Sep 17 00:00:00 2001 From: thotam Date: Sat, 6 Jun 2026 17:47:46 +0700 Subject: [PATCH] feat: support passing image URL directly to vision providers in read_image tool --- internal/providers/openai_request.go | 6 ++- internal/providers/openai_test.go | 25 +++++++++++++ internal/providers/types.go | 3 +- internal/tools/read_image.go | 28 +++++++++++++- internal/tools/read_image_test.go | 55 ++++++++++++++++++++++++++++ 5 files changed, 113 insertions(+), 4 deletions(-) create mode 100644 internal/tools/read_image_test.go diff --git a/internal/providers/openai_request.go b/internal/providers/openai_request.go index 401d6dcc..f60501e9 100644 --- a/internal/providers/openai_request.go +++ b/internal/providers/openai_request.go @@ -74,10 +74,14 @@ func (p *OpenAIProvider) buildRequestBody(model string, req ChatRequest, stream }) } for _, img := range m.Images { + urlVal := img.URL + if urlVal == "" { + urlVal = fmt.Sprintf("data:%s;base64,%s", img.MimeType, img.Data) + } parts = append(parts, map[string]any{ "type": "image_url", "image_url": map[string]any{ - "url": fmt.Sprintf("data:%s;base64,%s", img.MimeType, img.Data), + "url": urlVal, }, }) } diff --git a/internal/providers/openai_test.go b/internal/providers/openai_test.go index ccf6528d..52dff30d 100644 --- a/internal/providers/openai_test.go +++ b/internal/providers/openai_test.go @@ -240,6 +240,31 @@ func TestBuildRequestBody_MultimodalTextBeforeImages(t *testing.T) { } } +func TestBuildRequestBody_MultimodalWithImageURL(t *testing.T) { + p := NewOpenAIProvider("test", "key", "https://api.openai.com/v1", "gpt-4") + req := ChatRequest{ + Messages: []Message{ + { + Role: "user", + Content: "describe", + Images: []ImageContent{ + {URL: "https://example.com/image.png"}, + }, + }, + }, + } + body := p.buildRequestBody("gpt-4o", req, false) + msgs := body["messages"].([]map[string]any) + parts, ok := msgs[0]["content"].([]map[string]any) + if !ok || len(parts) < 2 { + t.Fatalf("want multimodal parts, got %v", msgs[0]["content"]) + } + imgPart := parts[1]["image_url"].(map[string]any) + if urlVal, _ := imgPart["url"].(string); urlVal != "https://example.com/image.png" { + t.Errorf("expected URL to be https://example.com/image.png, got %q", urlVal) + } +} + func TestBuildRequestBody_TogetherDetectedByProviderType(t *testing.T) { // Together behind reverse proxy — detected by providerType, not URL. p := NewOpenAIProvider("my-proxy", "key", "https://proxy.internal/v1", "") diff --git a/internal/providers/types.go b/internal/providers/types.go index d7f03413..70066373 100644 --- a/internal/providers/types.go +++ b/internal/providers/types.go @@ -113,10 +113,11 @@ type StreamChunk struct { Images []ImageContent `json:"images,omitempty"` // image generation frames (Codex) } -// ImageContent represents a base64-encoded image for vision-capable models. +// ImageContent represents an image (either base64-encoded or a direct URL) for vision-capable models. type ImageContent struct { MimeType string `json:"mime_type"` // e.g. "image/jpeg" Data string `json:"data"` // base64-encoded image bytes + URL string `json:"url,omitempty"` // URL of the image Partial bool `json:"partial,omitempty"` // true for intermediate frames (Codex image_generation_call) } diff --git a/internal/tools/read_image.go b/internal/tools/read_image.go index 892a6dfb..9c90d0fc 100644 --- a/internal/tools/read_image.go +++ b/internal/tools/read_image.go @@ -77,6 +77,10 @@ func (t *ReadImageTool) Parameters() map[string]any { "type": "string", "description": "Optional file path to an image in the workspace. Use this for generated images or attachments. If omitted, analyzes images from the conversation.", }, + "url": map[string]any{ + "type": "string", + "description": "Optional URL to an image. Use this to analyze images hosted online.", + }, }, "required": []string{"prompt"}, } @@ -91,18 +95,29 @@ func (t *ReadImageTool) Execute(ctx context.Context, args map[string]any) *Resul prompt = "Describe this image in detail." } + imgPath, _ := args["path"].(string) + imgURL, _ := args["url"].(string) + + if imgPath != "" && imgURL != "" { + return ErrorResult("Both 'path' and 'url' parameters cannot be specified. Choose only one.") + } + // If path is provided, load image from workspace file images := MediaImagesFromCtx(ctx) - if imgPath, _ := args["path"].(string); imgPath != "" { + if imgPath != "" { fileImages, err := t.loadImageFromPath(ctx, imgPath) if err != nil { return ErrorResult(err.Error()) } images = fileImages + } else if imgURL != "" { + images = []providers.ImageContent{{ + URL: imgURL, + }} } if len(images) == 0 { - return ErrorResult("No images available. Either send an image in the chat or provide a file path with the 'path' parameter.") + return ErrorResult("No images available. Either send an image in the chat, provide a file path with 'path', or provide an image URL with 'url'.") } chain := ResolveMediaProviderChain(ctx, "read_image", "", "", @@ -138,6 +153,15 @@ func (t *ReadImageTool) callProvider(ctx context.Context, cp credentialProvider, prompt := GetParamString(params, "prompt", "Describe this image in detail.") images, _ := params["images"].([]providers.ImageContent) + // Anthropic Claude does not support URL references and requires base64-encoded image data. + if providerName == "anthropic" || providerName == "claude-cli" { + for _, img := range images { + if img.URL != "" && img.Data == "" { + return nil, nil, fmt.Errorf("provider %q does not support analyzing images directly from a URL", providerName) + } + } + } + // Get the full provider for Chat() access p, err := t.registry.Get(ctx, providerName) if err != nil { diff --git a/internal/tools/read_image_test.go b/internal/tools/read_image_test.go new file mode 100644 index 00000000..1d61f86d --- /dev/null +++ b/internal/tools/read_image_test.go @@ -0,0 +1,55 @@ +package tools + +import ( + "context" + "strings" + "testing" + + "github.com/nextlevelbuilder/goclaw/internal/providers" +) + +func TestReadImage_BothPathAndUrl_Error(t *testing.T) { + tool := NewReadImageTool(nil) + + res := tool.Execute(context.Background(), map[string]any{ + "prompt": "describe this", + "path": "workspace/image.png", + "url": "https://example.com/image.png", + }) + + if !res.IsError { + t.Fatalf("expected error when both path and url are provided") + } + + if !strings.Contains(res.ForLLM, "Both 'path' and 'url' parameters cannot be specified") { + t.Errorf("unexpected error message: %s", res.ForLLM) + } +} + +func TestReadImage_AnthropicURL_Error(t *testing.T) { + tool := NewReadImageTool(nil) + + params := map[string]any{ + "prompt": "describe this", + "images": []providers.ImageContent{ + { + URL: "https://example.com/image.png", + }, + }, + } + + _, _, err := tool.callProvider(context.Background(), nil, "anthropic", "claude-3-sonnet", params) + if err == nil { + t.Fatalf("expected error for anthropic provider with image URL") + } + + if !strings.Contains(err.Error(), "does not support analyzing images directly from a URL") { + t.Errorf("unexpected error message: %v", err) + } + + // Should also error for claude-cli + _, _, err = tool.callProvider(context.Background(), nil, "claude-cli", "claude-3-sonnet", params) + if err == nil { + t.Fatalf("expected error for claude-cli provider with image URL") + } +}