From b8afe6a89944af6badef1195f348a329ce90e0ef Mon Sep 17 00:00:00 2001 From: viettranx Date: Wed, 4 Mar 2026 20:08:59 +0700 Subject: [PATCH] feat(security): add web_fetch domain blocklist, enhance IP scrubbing, enforce delegation scope - Add blocked_domains to web_fetch policy (always enforced regardless of allow_all/allowlist mode) - Refactor domain matching into shared matchDomainList() for allowlist and blocklist - Enhance server IP scrubbing: register decimal IP, dashed pattern, and reverse DNS hostnames - Add SOUL.md scope enforcement to delegation ExtraPrompt so agents refuse out-of-scope tasks - Add blocked domains UI textarea in dashboard tools config Co-Authored-By: Claude Opus 4.6 --- cmd/gateway.go | 5 +- internal/config/config_channels.go | 5 +- internal/tools/delegate.go | 3 +- internal/tools/scrub_server.go | 52 +++++++++++++++++-- internal/tools/web_fetch.go | 47 ++++++++++++----- .../pages/config/sections/tools-section.tsx | 14 +++++ 6 files changed, 104 insertions(+), 22 deletions(-) diff --git a/cmd/gateway.go b/cmd/gateway.go index f073531a..2eee6892 100644 --- a/cmd/gateway.go +++ b/cmd/gateway.go @@ -186,9 +186,10 @@ func runGateway() { webFetchTool := tools.NewWebFetchTool(tools.WebFetchConfig{ Policy: cfg.Tools.WebFetch.Policy, AllowedDomains: cfg.Tools.WebFetch.AllowedDomains, + BlockedDomains: cfg.Tools.WebFetch.BlockedDomains, }) toolsReg.Register(webFetchTool) - slog.Info("web_fetch tool enabled", "policy", cfg.Tools.WebFetch.Policy) + slog.Info("web_fetch tool enabled", "policy", cfg.Tools.WebFetch.Policy, "blocked", len(cfg.Tools.WebFetch.BlockedDomains)) // Vision fallback tool (for non-vision providers like MiniMax) toolsReg.Register(tools.NewReadImageTool(providerRegistry)) @@ -955,7 +956,7 @@ func runGateway() { if !ok { return } - webFetchTool.UpdatePolicy(updatedCfg.Tools.WebFetch.Policy, updatedCfg.Tools.WebFetch.AllowedDomains) + webFetchTool.UpdatePolicy(updatedCfg.Tools.WebFetch.Policy, updatedCfg.Tools.WebFetch.AllowedDomains, updatedCfg.Tools.WebFetch.BlockedDomains) }) go consumeInboundMessages(ctx, msgBus, agentRouter, cfg, sched, channelMgr, consumerTeamStore, quotaChecker) diff --git a/internal/config/config_channels.go b/internal/config/config_channels.go index 8fef3fe9..53b57a4a 100644 --- a/internal/config/config_channels.go +++ b/internal/config/config_channels.go @@ -256,8 +256,9 @@ type ExecApprovalCfg struct { // WebFetchPolicyConfig controls domain filtering for the web_fetch tool. type WebFetchPolicyConfig struct { - Policy string `json:"policy,omitempty"` // "allow_all" (default), "allowlist" - AllowedDomains []string `json:"allowed_domains,omitempty"` // e.g. ["github.com", "*.example.com"] + Policy string `json:"policy,omitempty"` // "allow_all" (default), "allowlist" + AllowedDomains []string `json:"allowed_domains,omitempty"` // e.g. ["github.com", "*.example.com"] + BlockedDomains []string `json:"blocked_domains,omitempty"` // always checked regardless of policy } // BrowserToolConfig controls the browser automation tool. diff --git a/internal/tools/delegate.go b/internal/tools/delegate.go index 51a5cb80..b4f683a9 100644 --- a/internal/tools/delegate.go +++ b/internal/tools/delegate.go @@ -671,6 +671,7 @@ func (dm *DelegateManager) buildRunRequest(task *DelegationTask, message string) "- Your complete response will be returned to the requesting agent.\n" + "- Do NOT try to communicate with the end user directly.\n" + "- Do NOT use your persona name or self-references (e.g. do not say your name). Write factual, neutral content.\n" + - "- Be concise and deliver actionable results.", + "- Be concise and deliver actionable results.\n" + + "- IMPORTANT: If the delegated task falls outside your expertise scope (as defined in your SOUL.md), politely refuse and explain that this task is not within your domain. Do NOT attempt tasks outside your scope.", } } diff --git a/internal/tools/scrub_server.go b/internal/tools/scrub_server.go index 76298e75..c4a8e203 100644 --- a/internal/tools/scrub_server.go +++ b/internal/tools/scrub_server.go @@ -2,6 +2,7 @@ package tools import ( "context" + "fmt" "io" "log/slog" "net" @@ -31,18 +32,59 @@ func DetectServerIPs(ctx context.Context) { "count", len(localIPs), "ips", localIPs) } - // Phase 2: Public IP (async, HTTP call) + // Phase 2: Public IP + derived values (async, HTTP + DNS calls) go func() { pubIP := detectPublicIP(ctx) - if pubIP != "" { - AddDynamicScrubValues(pubIP) - slog.Info("security.server_ip_scrub: public IP registered", "ip", pubIP) - } else { + if pubIP == "" { slog.Warn("security.server_ip_scrub: public IP detection failed") + return + } + AddDynamicScrubValues(pubIP) + slog.Info("security.server_ip_scrub: public IP registered", "ip", pubIP) + + derived := deriveIPScrubValues(pubIP) + if len(derived) > 0 { + AddDynamicScrubValues(derived...) + slog.Info("security.server_ip_scrub: derived values registered", + "count", len(derived), "values", derived) } }() } +// deriveIPScrubValues computes additional scrub values from an IP address: +// decimal representation, reverse DNS hostnames, and IP-with-dashes pattern. +func deriveIPScrubValues(ip string) []string { + var derived []string + + parsed := net.ParseIP(ip) + if parsed == nil { + return nil + } + + // Decimal representation (IPv4 only) + if v4 := parsed.To4(); v4 != nil { + decimal := uint32(v4[0])<<24 | uint32(v4[1])<<16 | uint32(v4[2])<<8 | uint32(v4[3]) + derived = append(derived, fmt.Sprintf("%d", decimal)) + } + + // IP-with-dashes pattern (e.g. "18-141-232-136" from ec2-style hostnames) + dashed := strings.ReplaceAll(ip, ".", "-") + derived = append(derived, dashed) + + // Reverse DNS hostnames (timeout 3s via default resolver) + names, err := net.LookupAddr(ip) + if err == nil { + for _, name := range names { + name = strings.TrimSuffix(name, ".") + if name != "" { + derived = append(derived, name) + } + } + } + + return derived +} + // detectLocalIPs returns non-loopback, non-link-local IPs from local interfaces. func detectLocalIPs() []string { addrs, err := net.InterfaceAddrs() diff --git a/internal/tools/web_fetch.go b/internal/tools/web_fetch.go index 243e0d37..cf354488 100644 --- a/internal/tools/web_fetch.go +++ b/internal/tools/web_fetch.go @@ -27,6 +27,7 @@ type WebFetchTool struct { cache *webCache policy string // "allow_all" (default), "allowlist" allowedDomains []string // domains when policy="allowlist" (supports "*.example.com") + blockedDomains []string // always checked regardless of policy (supports "*.example.com") mu sync.RWMutex } @@ -36,6 +37,7 @@ type WebFetchConfig struct { CacheTTL time.Duration Policy string // "allow_all" (default), "allowlist" AllowedDomains []string // domains when policy="allowlist" + BlockedDomains []string // always blocked regardless of policy } func NewWebFetchTool(cfg WebFetchConfig) *WebFetchTool { @@ -56,30 +58,28 @@ func NewWebFetchTool(cfg WebFetchConfig) *WebFetchTool { cache: newWebCache(defaultCacheMaxEntries, ttl), policy: policy, allowedDomains: cfg.AllowedDomains, + blockedDomains: cfg.BlockedDomains, } } // UpdatePolicy replaces the domain policy at runtime (called via pub/sub on config change). -func (t *WebFetchTool) UpdatePolicy(policy string, domains []string) { +func (t *WebFetchTool) UpdatePolicy(policy string, allowed, blocked []string) { t.mu.Lock() defer t.mu.Unlock() if policy == "" { policy = "allow_all" } t.policy = policy - t.allowedDomains = domains - slog.Info("web_fetch policy updated", "policy", policy, "domains", len(domains)) + t.allowedDomains = allowed + t.blockedDomains = blocked + slog.Info("web_fetch policy updated", "policy", policy, "allowed", len(allowed), "blocked", len(blocked)) } -// isDomainAllowed checks if a hostname matches the allowlist. +// matchDomainList checks if a hostname matches any pattern in the list. // Supports exact match ("github.com") and wildcard prefix ("*.example.com"). -func (t *WebFetchTool) isDomainAllowed(hostname string) bool { - t.mu.RLock() - domains := t.allowedDomains - t.mu.RUnlock() - +func matchDomainList(hostname string, patterns []string) bool { hostname = strings.ToLower(hostname) - for _, pattern := range domains { + for _, pattern := range patterns { pattern = strings.ToLower(strings.TrimSpace(pattern)) if pattern == hostname { return true @@ -95,6 +95,22 @@ func (t *WebFetchTool) isDomainAllowed(hostname string) bool { return false } +// isDomainAllowed checks if a hostname matches the allowlist. +func (t *WebFetchTool) isDomainAllowed(hostname string) bool { + t.mu.RLock() + domains := t.allowedDomains + t.mu.RUnlock() + return matchDomainList(hostname, domains) +} + +// isDomainBlocked checks if a hostname matches the blocklist. +func (t *WebFetchTool) isDomainBlocked(hostname string) bool { + t.mu.RLock() + domains := t.blockedDomains + t.mu.RUnlock() + return matchDomainList(hostname, domains) +} + func (t *WebFetchTool) Name() string { return "web_fetch" } func (t *WebFetchTool) Description() string { @@ -147,12 +163,19 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]interface{}) return ErrorResult(fmt.Sprintf("SSRF protection: %v", err)) } + hostname := parsed.Hostname() + + // Domain blocklist check (always enforced regardless of policy) + if t.isDomainBlocked(hostname) { + return ErrorResult(fmt.Sprintf("domain %q is blocked by policy", hostname)) + } + // Domain allowlist check t.mu.RLock() policy := t.policy t.mu.RUnlock() - if policy == "allowlist" && !t.isDomainAllowed(parsed.Hostname()) { - return ErrorResult(fmt.Sprintf("domain %q is not in the allowed domains list", parsed.Hostname())) + if policy == "allowlist" && !t.isDomainAllowed(hostname) { + return ErrorResult(fmt.Sprintf("domain %q is not in the allowed domains list", hostname)) } extractMode := "markdown" diff --git a/ui/web/src/pages/config/sections/tools-section.tsx b/ui/web/src/pages/config/sections/tools-section.tsx index 8735e9ab..68799a12 100644 --- a/ui/web/src/pages/config/sections/tools-section.tsx +++ b/ui/web/src/pages/config/sections/tools-section.tsx @@ -259,6 +259,20 @@ export function ToolsSection({ data, onSave, saving }: Props) { /> )} +
+ Blocked Domains (one per line, supports *.example.com) +