From 0592be359dcaf5182d2d0a380ba9fa23bc942b01 Mon Sep 17 00:00:00 2001 From: Luan Vu Date: Wed, 11 Mar 2026 17:37:55 +0700 Subject: [PATCH] fix: remove legacy per-agent imageGen/vision override from tools_config (#153) The per-agent `imageGen` and `vision` fields in `ToolPolicySpec` (stored in agents.tools_config JSONB) were added in d5cc5a7 (Feb 26) as the original way to configure image/vision providers. When the media provider chain system was introduced in 5815437 (Mar 8), these fields were kept "for backward compat" but became dead code with no UI to manage them. This causes a hard-to-debug issue: if an agent's tools_config contains stale imageGen/vision data (set via API or leftover from DB), it silently overrides the global provider chain configured in the builtin tools UI. Users see the correct chain in the UI but the tool calls a completely different provider/model, with no indication of why. Changes: - Remove Vision and ImageGen fields + struct definitions from ToolPolicySpec - Remove associated context helpers (WithVisionConfig, WithImageGenConfig, etc.) - Remove per-agent override injection in agent loop - Simplify create_image and read_image to use chain as sole source of truth - UI: whitelist known tools_config fields on save to clean stale DB data Co-authored-by: Luvu182 <208665161+Luvu182@users.noreply.github.com> --- internal/agent/loop.go | 11 +------- internal/config/config_channels.go | 16 ------------ internal/tools/context_keys.go | 25 ------------------- internal/tools/create_image.go | 9 +------ internal/tools/read_image.go | 9 +------ .../agents/agent-detail/agent-config-tab.tsx | 4 ++- 6 files changed, 6 insertions(+), 68 deletions(-) diff --git a/internal/agent/loop.go b/internal/agent/loop.go index be75bcf4..f4029ed9 100644 --- a/internal/agent/loop.go +++ b/internal/agent/loop.go @@ -56,16 +56,7 @@ func (l *Loop) runLoop(ctx context.Context, req RunRequest) (*RunResult, error) if req.SenderID != "" { ctx = store.WithSenderID(ctx, req.SenderID) } - // Inject per-agent vision/imagegen config for read_image/create_image tools - if l.agentToolPolicy != nil { - if l.agentToolPolicy.Vision != nil { - ctx = tools.WithVisionConfig(ctx, l.agentToolPolicy.Vision) - } - if l.agentToolPolicy.ImageGen != nil { - ctx = tools.WithImageGenConfig(ctx, l.agentToolPolicy.ImageGen) - } - } - // Inject global builtin tool settings (DB-level defaults, lower priority than per-agent) + // Inject global builtin tool settings for media tools (provider chain) if l.builtinToolSettings != nil { ctx = tools.WithBuiltinToolSettings(ctx, l.builtinToolSettings) } diff --git a/internal/config/config_channels.go b/internal/config/config_channels.go index e63d4848..e0afb410 100644 --- a/internal/config/config_channels.go +++ b/internal/config/config_channels.go @@ -341,22 +341,6 @@ type ToolPolicySpec struct { Deny []string `json:"deny,omitempty"` AlsoAllow []string `json:"alsoAllow,omitempty"` ByProvider map[string]*ToolPolicySpec `json:"byProvider,omitempty"` - Vision *VisionConfig `json:"vision,omitempty"` // per-agent vision provider/model override - ImageGen *ImageGenConfig `json:"imageGen,omitempty"` // per-agent image generation config -} - -// VisionConfig configures the provider and model for vision tools (read_image). -type VisionConfig struct { - Provider string `json:"provider,omitempty"` // e.g. "gemini", "anthropic" - Model string `json:"model,omitempty"` // e.g. "gemini-2.0-flash" -} - -// ImageGenConfig configures the provider and model for image generation (create_image). -type ImageGenConfig struct { - Provider string `json:"provider,omitempty"` // provider with image gen API (e.g. "openrouter") - Model string `json:"model,omitempty"` // e.g. "google/gemini-2.5-flash-image-preview" - Size string `json:"size,omitempty"` // default aspect ratio / size - Quality string `json:"quality,omitempty"` // "standard" or "hd" } type WebToolsConfig struct { diff --git a/internal/tools/context_keys.go b/internal/tools/context_keys.go index 555d2ec3..bacb1d3f 100644 --- a/internal/tools/context_keys.go +++ b/internal/tools/context_keys.go @@ -125,31 +125,6 @@ func ToolSessionKeyFromCtx(ctx context.Context) string { return v } -// --- Vision / ImageGen config (per-agent overrides) --- - -const ( - ctxVisionConfig toolContextKey = "tool_vision_config" - ctxImageGenConfig toolContextKey = "tool_imagegen_config" -) - -func WithVisionConfig(ctx context.Context, cfg *config.VisionConfig) context.Context { - return context.WithValue(ctx, ctxVisionConfig, cfg) -} - -func VisionConfigFromCtx(ctx context.Context) *config.VisionConfig { - v, _ := ctx.Value(ctxVisionConfig).(*config.VisionConfig) - return v -} - -func WithImageGenConfig(ctx context.Context, cfg *config.ImageGenConfig) context.Context { - return context.WithValue(ctx, ctxImageGenConfig, cfg) -} - -func ImageGenConfigFromCtx(ctx context.Context) *config.ImageGenConfig { - v, _ := ctx.Value(ctxImageGenConfig).(*config.ImageGenConfig) - return v -} - // --- Builtin tool settings (global DB overrides) --- const ctxBuiltinToolSettings toolContextKey = "tool_builtin_settings" diff --git a/internal/tools/create_image.go b/internal/tools/create_image.go index 67a81c64..6d8d1c90 100644 --- a/internal/tools/create_image.go +++ b/internal/tools/create_image.go @@ -78,14 +78,7 @@ func (t *CreateImageTool) Execute(ctx context.Context, args map[string]any) *Res aspectRatio = "1:1" } - // Extract per-agent config for backward compat - var perAgentProvider, perAgentModel string - if cfg := ImageGenConfigFromCtx(ctx); cfg != nil { - perAgentProvider = cfg.Provider - perAgentModel = cfg.Model - } - - chain := ResolveMediaProviderChain(ctx, "create_image", perAgentProvider, perAgentModel, + chain := ResolveMediaProviderChain(ctx, "create_image", "", "", imageGenProviderPriority, imageGenModelDefaults, t.registry) // Inject prompt and aspect_ratio into each chain entry's params diff --git a/internal/tools/read_image.go b/internal/tools/read_image.go index 980efa3e..b1bbcacc 100644 --- a/internal/tools/read_image.go +++ b/internal/tools/read_image.go @@ -75,14 +75,7 @@ func (t *ReadImageTool) Execute(ctx context.Context, args map[string]any) *Resul return ErrorResult("No images available in this conversation. The user may not have sent an image.") } - // Extract per-agent config for backward compat - var perAgentProvider, perAgentModel string - if cfg := VisionConfigFromCtx(ctx); cfg != nil { - perAgentProvider = cfg.Provider - perAgentModel = cfg.Model - } - - chain := ResolveMediaProviderChain(ctx, "read_image", perAgentProvider, perAgentModel, + chain := ResolveMediaProviderChain(ctx, "read_image", "", "", visionProviderPriority, visionModelDefaults, t.registry) // Inject prompt and images into each chain entry's params diff --git a/ui/web/src/pages/agents/agent-detail/agent-config-tab.tsx b/ui/web/src/pages/agents/agent-detail/agent-config-tab.tsx index 93d3b4d2..15c3b4e7 100644 --- a/ui/web/src/pages/agents/agent-detail/agent-config-tab.tsx +++ b/ui/web/src/pages/agents/agent-detail/agent-config-tab.tsx @@ -77,7 +77,9 @@ export function AgentConfigTab({ agent, onUpdate }: AgentConfigTabProps) { try { const updates: Record = { subagents_config: subEnabled ? sub : null, - tools_config: toolsEnabled ? tools : {}, + tools_config: toolsEnabled + ? { profile: tools.profile, allow: tools.allow, deny: tools.deny, alsoAllow: tools.alsoAllow } + : {}, compaction_config: comp, context_pruning: pruneEnabled ? prune : null, sandbox_config: sbEnabled ? sb : null,