mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-14 22:17:36 +00:00
feat(security): add web_fetch domain blocklist, enhance IP scrubbing, enforce delegation scope
- Add blocked_domains to web_fetch policy (always enforced regardless of allow_all/allowlist mode) - Refactor domain matching into shared matchDomainList() for allowlist and blocklist - Enhance server IP scrubbing: register decimal IP, dashed pattern, and reverse DNS hostnames - Add SOUL.md scope enforcement to delegation ExtraPrompt so agents refuse out-of-scope tasks - Add blocked domains UI textarea in dashboard tools config Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
b35dc56aff
commit
b8afe6a899
6 files changed
+104
-22
No files matched your search
+3
-2
@@ -186,9 +186,10 @@ func runGateway() {
|
||||
webFetchTool := tools.NewWebFetchTool(tools.WebFetchConfig{
|
||||
Policy: cfg.Tools.WebFetch.Policy,
|
||||
AllowedDomains: cfg.Tools.WebFetch.AllowedDomains,
|
||||
BlockedDomains: cfg.Tools.WebFetch.BlockedDomains,
|
||||
})
|
||||
toolsReg.Register(webFetchTool)
|
||||
slog.Info("web_fetch tool enabled", "policy", cfg.Tools.WebFetch.Policy)
|
||||
slog.Info("web_fetch tool enabled", "policy", cfg.Tools.WebFetch.Policy, "blocked", len(cfg.Tools.WebFetch.BlockedDomains))
|
||||
|
||||
// Vision fallback tool (for non-vision providers like MiniMax)
|
||||
toolsReg.Register(tools.NewReadImageTool(providerRegistry))
|
||||
@@ -955,7 +956,7 @@ func runGateway() {
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
webFetchTool.UpdatePolicy(updatedCfg.Tools.WebFetch.Policy, updatedCfg.Tools.WebFetch.AllowedDomains)
|
||||
webFetchTool.UpdatePolicy(updatedCfg.Tools.WebFetch.Policy, updatedCfg.Tools.WebFetch.AllowedDomains, updatedCfg.Tools.WebFetch.BlockedDomains)
|
||||
})
|
||||
|
||||
go consumeInboundMessages(ctx, msgBus, agentRouter, cfg, sched, channelMgr, consumerTeamStore, quotaChecker)
|
||||
|
||||
@@ -256,8 +256,9 @@ type ExecApprovalCfg struct {
|
||||
|
||||
// WebFetchPolicyConfig controls domain filtering for the web_fetch tool.
|
||||
type WebFetchPolicyConfig struct {
|
||||
Policy string `json:"policy,omitempty"` // "allow_all" (default), "allowlist"
|
||||
AllowedDomains []string `json:"allowed_domains,omitempty"` // e.g. ["github.com", "*.example.com"]
|
||||
Policy string `json:"policy,omitempty"` // "allow_all" (default), "allowlist"
|
||||
AllowedDomains []string `json:"allowed_domains,omitempty"` // e.g. ["github.com", "*.example.com"]
|
||||
BlockedDomains []string `json:"blocked_domains,omitempty"` // always checked regardless of policy
|
||||
}
|
||||
|
||||
// BrowserToolConfig controls the browser automation tool.
|
||||
|
||||
@@ -671,6 +671,7 @@ func (dm *DelegateManager) buildRunRequest(task *DelegationTask, message string)
|
||||
"- Your complete response will be returned to the requesting agent.\n" +
|
||||
"- Do NOT try to communicate with the end user directly.\n" +
|
||||
"- Do NOT use your persona name or self-references (e.g. do not say your name). Write factual, neutral content.\n" +
|
||||
"- Be concise and deliver actionable results.",
|
||||
"- Be concise and deliver actionable results.\n" +
|
||||
"- IMPORTANT: If the delegated task falls outside your expertise scope (as defined in your SOUL.md), politely refuse and explain that this task is not within your domain. Do NOT attempt tasks outside your scope.",
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
@@ -31,18 +32,59 @@ func DetectServerIPs(ctx context.Context) {
|
||||
"count", len(localIPs), "ips", localIPs)
|
||||
}
|
||||
|
||||
// Phase 2: Public IP (async, HTTP call)
|
||||
// Phase 2: Public IP + derived values (async, HTTP + DNS calls)
|
||||
go func() {
|
||||
pubIP := detectPublicIP(ctx)
|
||||
if pubIP != "" {
|
||||
AddDynamicScrubValues(pubIP)
|
||||
slog.Info("security.server_ip_scrub: public IP registered", "ip", pubIP)
|
||||
} else {
|
||||
if pubIP == "" {
|
||||
slog.Warn("security.server_ip_scrub: public IP detection failed")
|
||||
return
|
||||
}
|
||||
AddDynamicScrubValues(pubIP)
|
||||
slog.Info("security.server_ip_scrub: public IP registered", "ip", pubIP)
|
||||
|
||||
derived := deriveIPScrubValues(pubIP)
|
||||
if len(derived) > 0 {
|
||||
AddDynamicScrubValues(derived...)
|
||||
slog.Info("security.server_ip_scrub: derived values registered",
|
||||
"count", len(derived), "values", derived)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// deriveIPScrubValues computes additional scrub values from an IP address:
|
||||
// decimal representation, reverse DNS hostnames, and IP-with-dashes pattern.
|
||||
func deriveIPScrubValues(ip string) []string {
|
||||
var derived []string
|
||||
|
||||
parsed := net.ParseIP(ip)
|
||||
if parsed == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Decimal representation (IPv4 only)
|
||||
if v4 := parsed.To4(); v4 != nil {
|
||||
decimal := uint32(v4[0])<<24 | uint32(v4[1])<<16 | uint32(v4[2])<<8 | uint32(v4[3])
|
||||
derived = append(derived, fmt.Sprintf("%d", decimal))
|
||||
}
|
||||
|
||||
// IP-with-dashes pattern (e.g. "18-141-232-136" from ec2-style hostnames)
|
||||
dashed := strings.ReplaceAll(ip, ".", "-")
|
||||
derived = append(derived, dashed)
|
||||
|
||||
// Reverse DNS hostnames (timeout 3s via default resolver)
|
||||
names, err := net.LookupAddr(ip)
|
||||
if err == nil {
|
||||
for _, name := range names {
|
||||
name = strings.TrimSuffix(name, ".")
|
||||
if name != "" {
|
||||
derived = append(derived, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return derived
|
||||
}
|
||||
|
||||
// detectLocalIPs returns non-loopback, non-link-local IPs from local interfaces.
|
||||
func detectLocalIPs() []string {
|
||||
addrs, err := net.InterfaceAddrs()
|
||||
|
||||
+35
-12
@@ -27,6 +27,7 @@ type WebFetchTool struct {
|
||||
cache *webCache
|
||||
policy string // "allow_all" (default), "allowlist"
|
||||
allowedDomains []string // domains when policy="allowlist" (supports "*.example.com")
|
||||
blockedDomains []string // always checked regardless of policy (supports "*.example.com")
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
@@ -36,6 +37,7 @@ type WebFetchConfig struct {
|
||||
CacheTTL time.Duration
|
||||
Policy string // "allow_all" (default), "allowlist"
|
||||
AllowedDomains []string // domains when policy="allowlist"
|
||||
BlockedDomains []string // always blocked regardless of policy
|
||||
}
|
||||
|
||||
func NewWebFetchTool(cfg WebFetchConfig) *WebFetchTool {
|
||||
@@ -56,30 +58,28 @@ func NewWebFetchTool(cfg WebFetchConfig) *WebFetchTool {
|
||||
cache: newWebCache(defaultCacheMaxEntries, ttl),
|
||||
policy: policy,
|
||||
allowedDomains: cfg.AllowedDomains,
|
||||
blockedDomains: cfg.BlockedDomains,
|
||||
}
|
||||
}
|
||||
|
||||
// UpdatePolicy replaces the domain policy at runtime (called via pub/sub on config change).
|
||||
func (t *WebFetchTool) UpdatePolicy(policy string, domains []string) {
|
||||
func (t *WebFetchTool) UpdatePolicy(policy string, allowed, blocked []string) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if policy == "" {
|
||||
policy = "allow_all"
|
||||
}
|
||||
t.policy = policy
|
||||
t.allowedDomains = domains
|
||||
slog.Info("web_fetch policy updated", "policy", policy, "domains", len(domains))
|
||||
t.allowedDomains = allowed
|
||||
t.blockedDomains = blocked
|
||||
slog.Info("web_fetch policy updated", "policy", policy, "allowed", len(allowed), "blocked", len(blocked))
|
||||
}
|
||||
|
||||
// isDomainAllowed checks if a hostname matches the allowlist.
|
||||
// matchDomainList checks if a hostname matches any pattern in the list.
|
||||
// Supports exact match ("github.com") and wildcard prefix ("*.example.com").
|
||||
func (t *WebFetchTool) isDomainAllowed(hostname string) bool {
|
||||
t.mu.RLock()
|
||||
domains := t.allowedDomains
|
||||
t.mu.RUnlock()
|
||||
|
||||
func matchDomainList(hostname string, patterns []string) bool {
|
||||
hostname = strings.ToLower(hostname)
|
||||
for _, pattern := range domains {
|
||||
for _, pattern := range patterns {
|
||||
pattern = strings.ToLower(strings.TrimSpace(pattern))
|
||||
if pattern == hostname {
|
||||
return true
|
||||
@@ -95,6 +95,22 @@ func (t *WebFetchTool) isDomainAllowed(hostname string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// isDomainAllowed checks if a hostname matches the allowlist.
|
||||
func (t *WebFetchTool) isDomainAllowed(hostname string) bool {
|
||||
t.mu.RLock()
|
||||
domains := t.allowedDomains
|
||||
t.mu.RUnlock()
|
||||
return matchDomainList(hostname, domains)
|
||||
}
|
||||
|
||||
// isDomainBlocked checks if a hostname matches the blocklist.
|
||||
func (t *WebFetchTool) isDomainBlocked(hostname string) bool {
|
||||
t.mu.RLock()
|
||||
domains := t.blockedDomains
|
||||
t.mu.RUnlock()
|
||||
return matchDomainList(hostname, domains)
|
||||
}
|
||||
|
||||
func (t *WebFetchTool) Name() string { return "web_fetch" }
|
||||
|
||||
func (t *WebFetchTool) Description() string {
|
||||
@@ -147,12 +163,19 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]interface{})
|
||||
return ErrorResult(fmt.Sprintf("SSRF protection: %v", err))
|
||||
}
|
||||
|
||||
hostname := parsed.Hostname()
|
||||
|
||||
// Domain blocklist check (always enforced regardless of policy)
|
||||
if t.isDomainBlocked(hostname) {
|
||||
return ErrorResult(fmt.Sprintf("domain %q is blocked by policy", hostname))
|
||||
}
|
||||
|
||||
// Domain allowlist check
|
||||
t.mu.RLock()
|
||||
policy := t.policy
|
||||
t.mu.RUnlock()
|
||||
if policy == "allowlist" && !t.isDomainAllowed(parsed.Hostname()) {
|
||||
return ErrorResult(fmt.Sprintf("domain %q is not in the allowed domains list", parsed.Hostname()))
|
||||
if policy == "allowlist" && !t.isDomainAllowed(hostname) {
|
||||
return ErrorResult(fmt.Sprintf("domain %q is not in the allowed domains list", hostname))
|
||||
}
|
||||
|
||||
extractMode := "markdown"
|
||||
|
||||
@@ -259,6 +259,20 @@ export function ToolsSection({ data, onSave, saving }: Props) {
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
<div className="grid gap-1.5">
|
||||
<InfoLabel tip="Domains that are always blocked regardless of policy mode. Useful for preventing info-leak sites (IP lookup, etc.).">Blocked Domains (one per line, supports *.example.com)</InfoLabel>
|
||||
<Textarea
|
||||
value={(webFetch.blocked_domains ?? []).join("\n")}
|
||||
onChange={(e) =>
|
||||
updateNested("web_fetch", {
|
||||
...webFetch,
|
||||
blocked_domains: e.target.value.split("\n").filter(Boolean),
|
||||
})
|
||||
}
|
||||
className="min-h-[80px] font-mono text-xs"
|
||||
placeholder={"ifconfig.co\nipinfo.io\n*.whatismyip.com"}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
Reference in new issue
Block a user