mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-14 20:17:20 +00:00
feat(tools): tenant-aware web_fetch domain policy resolution
web_fetch now reads per-tenant policy overrides from BuiltinToolSettingsFromCtx before falling back to the tool's default policy. Same per-call resolution pattern as web_search. - Add resolvePolicy(ctx) with webFetchPolicyOverride struct - Refactor Execute + doFetch + fetchRawContent to use webFetchPolicy - InProcessExtractor also resolves policy from ctx - Remove isDomainAllowed/isDomainBlocked (replaced by matchDomainList) - 6 new unit tests for tenant policy override scenarios
This commit is contained in:
1 parent
30a40bbe59
commit
def1712f3b
3 files changed
+186
-34
No files matched your search
+56
-29
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
@@ -79,6 +80,49 @@ func (t *WebFetchTool) UpdatePolicy(policy string, allowed, blocked []string) {
|
|||||||
slog.Info("web_fetch policy updated", "policy", policy, "allowed", len(allowed), "blocked", len(blocked))
|
slog.Info("web_fetch policy updated", "policy", policy, "allowed", len(allowed), "blocked", len(blocked))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// webFetchPolicy holds the resolved domain policy for a single request.
|
||||||
|
type webFetchPolicy struct {
|
||||||
|
mode string // "allow_all" | "allowlist"
|
||||||
|
allowedDomains []string
|
||||||
|
blockedDomains []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// webFetchPolicyOverride is the tenant settings shape for web_fetch
|
||||||
|
// (stored in builtin_tool_tenant_configs.settings).
|
||||||
|
type webFetchPolicyOverride struct {
|
||||||
|
Policy string `json:"policy,omitempty"`
|
||||||
|
AllowedDomains []string `json:"allowed_domains,omitempty"`
|
||||||
|
BlockedDomains []string `json:"blocked_domains,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolvePolicy returns the effective domain policy for this request.
|
||||||
|
// Checks tenant override via BuiltinToolSettingsFromCtx first; falls back
|
||||||
|
// to the tool's default policy when no override is present.
|
||||||
|
func (t *WebFetchTool) resolvePolicy(ctx context.Context) webFetchPolicy {
|
||||||
|
if settings := BuiltinToolSettingsFromCtx(ctx); settings != nil {
|
||||||
|
if raw, ok := settings["web_fetch"]; ok && len(raw) > 0 {
|
||||||
|
var override webFetchPolicyOverride
|
||||||
|
if err := json.Unmarshal(raw, &override); err != nil {
|
||||||
|
slog.Warn("web_fetch: failed to parse tenant override, using defaults", "error", err)
|
||||||
|
} else if override.Policy != "" {
|
||||||
|
return webFetchPolicy{
|
||||||
|
mode: override.Policy,
|
||||||
|
allowedDomains: override.AllowedDomains,
|
||||||
|
blockedDomains: override.BlockedDomains,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Fall back to tool defaults
|
||||||
|
t.mu.RLock()
|
||||||
|
defer t.mu.RUnlock()
|
||||||
|
return webFetchPolicy{
|
||||||
|
mode: t.policy,
|
||||||
|
allowedDomains: t.allowedDomains,
|
||||||
|
blockedDomains: t.blockedDomains,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// matchDomainList checks if a hostname matches any pattern in the list.
|
// matchDomainList checks if a hostname matches any pattern in the list.
|
||||||
// Supports exact match ("github.com") and wildcard prefix ("*.example.com").
|
// Supports exact match ("github.com") and wildcard prefix ("*.example.com").
|
||||||
func matchDomainList(hostname string, patterns []string) bool {
|
func matchDomainList(hostname string, patterns []string) bool {
|
||||||
@@ -99,22 +143,6 @@ func matchDomainList(hostname string, patterns []string) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// isDomainAllowed checks if a hostname matches the allowlist.
|
|
||||||
func (t *WebFetchTool) isDomainAllowed(hostname string) bool {
|
|
||||||
t.mu.RLock()
|
|
||||||
domains := t.allowedDomains
|
|
||||||
t.mu.RUnlock()
|
|
||||||
return matchDomainList(hostname, domains)
|
|
||||||
}
|
|
||||||
|
|
||||||
// isDomainBlocked checks if a hostname matches the blocklist.
|
|
||||||
func (t *WebFetchTool) isDomainBlocked(hostname string) bool {
|
|
||||||
t.mu.RLock()
|
|
||||||
domains := t.blockedDomains
|
|
||||||
t.mu.RUnlock()
|
|
||||||
return matchDomainList(hostname, domains)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *WebFetchTool) Name() string { return "web_fetch" }
|
func (t *WebFetchTool) Name() string { return "web_fetch" }
|
||||||
|
|
||||||
func (t *WebFetchTool) Description() string {
|
func (t *WebFetchTool) Description() string {
|
||||||
@@ -167,18 +195,17 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *Result
|
|||||||
return ErrorResult(fmt.Sprintf("SSRF protection: %v", err))
|
return ErrorResult(fmt.Sprintf("SSRF protection: %v", err))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Resolve domain policy (tenant override via ctx, or tool defaults)
|
||||||
|
pol := t.resolvePolicy(ctx)
|
||||||
hostname := parsed.Hostname()
|
hostname := parsed.Hostname()
|
||||||
|
|
||||||
// Domain blocklist check (always enforced regardless of policy)
|
// Domain blocklist check (always enforced regardless of policy)
|
||||||
if t.isDomainBlocked(hostname) {
|
if matchDomainList(hostname, pol.blockedDomains) {
|
||||||
return ErrorResult(fmt.Sprintf("domain %q is blocked by policy", hostname))
|
return ErrorResult(fmt.Sprintf("domain %q is blocked by policy", hostname))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Domain allowlist check
|
// Domain allowlist check
|
||||||
t.mu.RLock()
|
if pol.mode == "allowlist" && !matchDomainList(hostname, pol.allowedDomains) {
|
||||||
policy := t.policy
|
|
||||||
t.mu.RUnlock()
|
|
||||||
if policy == "allowlist" && !t.isDomainAllowed(hostname) {
|
|
||||||
return ErrorResult(fmt.Sprintf("domain %q is not in the allowed domains list", hostname))
|
return ErrorResult(fmt.Sprintf("domain %q is not in the allowed domains list", hostname))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -212,7 +239,7 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *Result
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Fetch
|
// Fetch
|
||||||
result, err := t.doFetch(ctx, rawURL, extractMode, maxChars, policy)
|
result, err := t.doFetch(ctx, rawURL, extractMode, maxChars, pol)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
errMsg := truncateStr(err.Error(), defaultErrorMaxChars)
|
errMsg := truncateStr(err.Error(), defaultErrorMaxChars)
|
||||||
return ErrorResult(fmt.Sprintf("fetch failed: %s", errMsg))
|
return ErrorResult(fmt.Sprintf("fetch failed: %s", errMsg))
|
||||||
@@ -223,7 +250,7 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *Result
|
|||||||
return NewResult(wrapped)
|
return NewResult(wrapped)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string, maxChars int, policy string) (string, error) {
|
func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string, maxChars int, pol webFetchPolicy) (string, error) {
|
||||||
// For markdown mode, use the extractor chain (Defuddle → InProcess waterfall)
|
// For markdown mode, use the extractor chain (Defuddle → InProcess waterfall)
|
||||||
// resolved from builtin_tools settings stored in context.
|
// resolved from builtin_tools settings stored in context.
|
||||||
// InProcessExtractor delegates to fetchRawContent (same path as doDirectFetch),
|
// InProcessExtractor delegates to fetchRawContent (same path as doDirectFetch),
|
||||||
@@ -240,7 +267,7 @@ func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string,
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Text mode or no chain available — use direct HTTP fetch.
|
// Text mode or no chain available — use direct HTTP fetch.
|
||||||
return t.doDirectFetch(ctx, rawURL, extractMode, maxChars, policy)
|
return t.doDirectFetch(ctx, rawURL, extractMode, maxChars, pol)
|
||||||
}
|
}
|
||||||
|
|
||||||
// fetchRawResult holds the output from fetchRawContent.
|
// fetchRawResult holds the output from fetchRawContent.
|
||||||
@@ -254,7 +281,7 @@ type fetchRawResult struct {
|
|||||||
// fetchRawContent performs HTTP GET with full security checks (SSRF, domain policy on
|
// fetchRawContent performs HTTP GET with full security checks (SSRF, domain policy on
|
||||||
// redirects) and routes content by type. Returns raw extracted content without formatting.
|
// redirects) and routes content by type. Returns raw extracted content without formatting.
|
||||||
// Used by both doDirectFetch (text mode) and InProcessExtractor (chain fallback).
|
// Used by both doDirectFetch (text mode) and InProcessExtractor (chain fallback).
|
||||||
func (t *WebFetchTool) fetchRawContent(ctx context.Context, rawURL, extractMode string, maxChars int, policy string) (fetchRawResult, error) {
|
func (t *WebFetchTool) fetchRawContent(ctx context.Context, rawURL, extractMode string, maxChars int, pol webFetchPolicy) (fetchRawResult, error) {
|
||||||
req, err := http.NewRequestWithContext(ctx, "GET", rawURL, nil)
|
req, err := http.NewRequestWithContext(ctx, "GET", rawURL, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fetchRawResult{}, fmt.Errorf("create request: %w", err)
|
return fetchRawResult{}, fmt.Errorf("create request: %w", err)
|
||||||
@@ -280,10 +307,10 @@ func (t *WebFetchTool) fetchRawContent(ctx context.Context, rawURL, extractMode
|
|||||||
return fmt.Errorf("redirect SSRF protection: %w", err)
|
return fmt.Errorf("redirect SSRF protection: %w", err)
|
||||||
}
|
}
|
||||||
redirectHost := req.URL.Hostname()
|
redirectHost := req.URL.Hostname()
|
||||||
if t.isDomainBlocked(redirectHost) {
|
if matchDomainList(redirectHost, pol.blockedDomains) {
|
||||||
return fmt.Errorf("redirect to %q blocked: domain is in blocklist", redirectHost)
|
return fmt.Errorf("redirect to %q blocked: domain is in blocklist", redirectHost)
|
||||||
}
|
}
|
||||||
if policy == "allowlist" && !t.isDomainAllowed(redirectHost) {
|
if pol.mode == "allowlist" && !matchDomainList(redirectHost, pol.allowedDomains) {
|
||||||
return fmt.Errorf("redirect to %q blocked: domain not in allowlist", redirectHost)
|
return fmt.Errorf("redirect to %q blocked: domain not in allowlist", redirectHost)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
@@ -348,8 +375,8 @@ func (t *WebFetchTool) fetchRawContent(ctx context.Context, rawURL, extractMode
|
|||||||
|
|
||||||
// doDirectFetch wraps fetchRawContent with full HTTP metadata formatting.
|
// doDirectFetch wraps fetchRawContent with full HTTP metadata formatting.
|
||||||
// Used for text mode extraction and as ultimate fallback.
|
// Used for text mode extraction and as ultimate fallback.
|
||||||
func (t *WebFetchTool) doDirectFetch(ctx context.Context, rawURL, extractMode string, maxChars int, policy string) (string, error) {
|
func (t *WebFetchTool) doDirectFetch(ctx context.Context, rawURL, extractMode string, maxChars int, pol webFetchPolicy) (string, error) {
|
||||||
raw, err := t.fetchRawContent(ctx, rawURL, extractMode, maxChars, policy)
|
raw, err := t.fetchRawContent(ctx, rawURL, extractMode, maxChars, pol)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,11 +16,8 @@ func (e *InProcessExtractor) Name() string { return "html-to-markdown" }
|
|||||||
// Extract fetches the URL via the tool's fetchRawContent (full security checks)
|
// Extract fetches the URL via the tool's fetchRawContent (full security checks)
|
||||||
// and returns the raw extracted markdown content.
|
// and returns the raw extracted markdown content.
|
||||||
func (e *InProcessExtractor) Extract(ctx context.Context, rawURL string) (string, error) {
|
func (e *InProcessExtractor) Extract(ctx context.Context, rawURL string) (string, error) {
|
||||||
e.tool.mu.RLock()
|
pol := e.tool.resolvePolicy(ctx)
|
||||||
policy := e.tool.policy
|
raw, err := e.tool.fetchRawContent(ctx, rawURL, "markdown", defaultFetchMaxChars, pol)
|
||||||
e.tool.mu.RUnlock()
|
|
||||||
|
|
||||||
raw, err := e.tool.fetchRawContent(ctx, rawURL, "markdown", defaultFetchMaxChars, policy)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,128 @@
|
|||||||
|
package tools
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ctxWithWebFetchSettings returns a ctx with builtin tool settings containing
|
||||||
|
// the given web_fetch policy override.
|
||||||
|
func ctxWithWebFetchSettings(t *testing.T, override webFetchPolicyOverride) context.Context {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := json.Marshal(override)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
settings := BuiltinToolSettings{"web_fetch": raw}
|
||||||
|
return WithBuiltinToolSettings(context.Background(), settings)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePolicy_NoOverride_ReturnsDefaults(t *testing.T) {
|
||||||
|
tool := NewWebFetchTool(WebFetchConfig{
|
||||||
|
Policy: "allow_all",
|
||||||
|
AllowedDomains: []string{"default.com"},
|
||||||
|
BlockedDomains: []string{"blocked.com"},
|
||||||
|
})
|
||||||
|
|
||||||
|
pol := tool.resolvePolicy(context.Background())
|
||||||
|
if pol.mode != "allow_all" {
|
||||||
|
t.Errorf("mode = %q, want allow_all", pol.mode)
|
||||||
|
}
|
||||||
|
if len(pol.blockedDomains) != 1 || pol.blockedDomains[0] != "blocked.com" {
|
||||||
|
t.Errorf("blockedDomains = %v, want [blocked.com]", pol.blockedDomains)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePolicy_TenantAllowlist(t *testing.T) {
|
||||||
|
tool := NewWebFetchTool(WebFetchConfig{
|
||||||
|
Policy: "allow_all",
|
||||||
|
AllowedDomains: []string{"default.com"},
|
||||||
|
})
|
||||||
|
|
||||||
|
ctx := ctxWithWebFetchSettings(t, webFetchPolicyOverride{
|
||||||
|
Policy: "allowlist",
|
||||||
|
AllowedDomains: []string{"tenant-a.com", "*.api.tenant-a.com"},
|
||||||
|
})
|
||||||
|
|
||||||
|
pol := tool.resolvePolicy(ctx)
|
||||||
|
if pol.mode != "allowlist" {
|
||||||
|
t.Errorf("mode = %q, want allowlist", pol.mode)
|
||||||
|
}
|
||||||
|
if len(pol.allowedDomains) != 2 {
|
||||||
|
t.Errorf("allowedDomains = %v, want 2 entries", pol.allowedDomains)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePolicy_TenantBlocklist(t *testing.T) {
|
||||||
|
tool := NewWebFetchTool(WebFetchConfig{
|
||||||
|
Policy: "allow_all",
|
||||||
|
BlockedDomains: []string{"global-blocked.com"},
|
||||||
|
})
|
||||||
|
|
||||||
|
ctx := ctxWithWebFetchSettings(t, webFetchPolicyOverride{
|
||||||
|
Policy: "allow_all",
|
||||||
|
BlockedDomains: []string{"tenant-blocked.com"},
|
||||||
|
})
|
||||||
|
|
||||||
|
pol := tool.resolvePolicy(ctx)
|
||||||
|
// Tenant override replaces global blocked list
|
||||||
|
if len(pol.blockedDomains) != 1 || pol.blockedDomains[0] != "tenant-blocked.com" {
|
||||||
|
t.Errorf("blockedDomains = %v, want [tenant-blocked.com]", pol.blockedDomains)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePolicy_MalformedJSON_FallsBack(t *testing.T) {
|
||||||
|
tool := NewWebFetchTool(WebFetchConfig{
|
||||||
|
Policy: "allow_all",
|
||||||
|
})
|
||||||
|
|
||||||
|
settings := BuiltinToolSettings{
|
||||||
|
"web_fetch": []byte(`{not valid json}`),
|
||||||
|
}
|
||||||
|
ctx := WithBuiltinToolSettings(context.Background(), settings)
|
||||||
|
|
||||||
|
pol := tool.resolvePolicy(ctx)
|
||||||
|
if pol.mode != "allow_all" {
|
||||||
|
t.Errorf("mode = %q, want allow_all (fallback)", pol.mode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePolicy_EmptyOverride_FallsBack(t *testing.T) {
|
||||||
|
tool := NewWebFetchTool(WebFetchConfig{
|
||||||
|
Policy: "allowlist",
|
||||||
|
AllowedDomains: []string{"default.com"},
|
||||||
|
})
|
||||||
|
|
||||||
|
// Override exists but has empty policy — should fall back to defaults
|
||||||
|
ctx := ctxWithWebFetchSettings(t, webFetchPolicyOverride{})
|
||||||
|
|
||||||
|
pol := tool.resolvePolicy(ctx)
|
||||||
|
if pol.mode != "allowlist" {
|
||||||
|
t.Errorf("mode = %q, want allowlist (default)", pol.mode)
|
||||||
|
}
|
||||||
|
if len(pol.allowedDomains) != 1 || pol.allowedDomains[0] != "default.com" {
|
||||||
|
t.Errorf("allowedDomains = %v, want [default.com]", pol.allowedDomains)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePolicy_TenantOverridesGlobal(t *testing.T) {
|
||||||
|
tool := NewWebFetchTool(WebFetchConfig{
|
||||||
|
Policy: "allowlist",
|
||||||
|
AllowedDomains: []string{"global.com"},
|
||||||
|
BlockedDomains: []string{"global-blocked.com"},
|
||||||
|
})
|
||||||
|
|
||||||
|
ctx := ctxWithWebFetchSettings(t, webFetchPolicyOverride{
|
||||||
|
Policy: "allow_all",
|
||||||
|
BlockedDomains: []string{"tenant-evil.com"},
|
||||||
|
})
|
||||||
|
|
||||||
|
pol := tool.resolvePolicy(ctx)
|
||||||
|
if pol.mode != "allow_all" {
|
||||||
|
t.Errorf("mode = %q, want allow_all (tenant override)", pol.mode)
|
||||||
|
}
|
||||||
|
if len(pol.blockedDomains) != 1 || pol.blockedDomains[0] != "tenant-evil.com" {
|
||||||
|
t.Errorf("blockedDomains = %v, want [tenant-evil.com]", pol.blockedDomains)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user