fix(vision): restore context fallback for read_image in file-ref mode

PR #511 removed WithMediaImages context in file-ref mode, breaking
read_image when LLM omits the path param. Restore base64 context as
fallback (costs Go memory, not LLM tokens). Also improve error messages
to distinguish missing provider config vs provider failures.
This commit is contained in:
viettranx committed 2026-03-27 15:59:42 +07:00
1 parent 7eea004c8d
commit 2f31db4c7e
2 files changed
+16 -5

No files matched your search

+11 -4
View File
@@ -90,7 +90,13 @@ func (l *Loop) runLoop(ctx context.Context, req RunRequest) (*RunResult, error)
}
}
if deferToReadImageTool {
// File-ref mode: skip base64 encoding entirely — images accessible via read_image(path=...).
// File-ref mode: images primarily accessed via read_image(path=...).
// Still load into context as fallback — if LLM omits the path param,
// read_image can fall back to context images. This costs Go memory
// but NOT LLM tokens (base64 is in Go context, not sent to provider).
if images := loadImages(imageFiles); len(images) > 0 {
ctx = tools.WithMediaImages(ctx, images)
}
slog.Info("vision: file-ref mode, images accessible via read_image tool",
"count", len(imageFiles), "agent", l.id)
} else if images := loadImages(imageFiles); len(images) > 0 {
@@ -101,9 +107,10 @@ func (l *Loop) runLoop(ctx context.Context, req RunRequest) (*RunResult, error)
}
}
// 2a. In inline mode, also load historical images into context for read_image tool.
// In file-ref mode (deferToReadImageTool), images are accessed via path — no base64 needed.
if !deferToReadImageTool && l.mediaStore != nil {
// 2a. Load historical images into context for read_image tool.
// Both modes need this: inline mode for main LLM, file-ref mode as fallback
// when LLM calls read_image without the path param.
if l.mediaStore != nil {
ctx = l.loadHistoricalImagesForTool(ctx, mediaRefs, messages)
}
+5 -1
View File
@@ -107,9 +107,13 @@ func (t *ReadImageTool) Execute(ctx context.Context, args map[string]any) *Resul
chain[i].Params["images"] = images
}
if len(chain) == 0 {
return ErrorResult("No vision provider configured. Ask the user to add a vision-capable provider (e.g. Gemini, Anthropic, OpenRouter) in the system settings.")
}
chainResult, err := ExecuteWithChain(ctx, chain, t.registry, t.callProvider)
if err != nil {
return ErrorResult(fmt.Sprintf("image analysis failed: %v", err))
return ErrorResult(fmt.Sprintf("Image analysis failed — all vision providers returned errors: %v. The user may need to check their provider API keys or configuration.", err))
}
result := NewResult(string(chainResult.Data))