feat(security): let operators un-block CIDRs behind a transparent proxy (#1465)

SSRF protection resolves a hostname and judges the resulting IP. That
model assumes DNS resolution describes where the traffic actually goes,
which stops being true behind a TUN/fake-IP proxy: every query is answered
with a synthetic address out of a reserved range, and the proxy then
routes that address to the real public host. The IP is a handle, not a
destination.

In that environment web_fetch rejects ordinary public sites — observed
with news.sina.cn resolving to 198.18.0.236 — and no configuration can
fix it, because the block list is compiled in. The agent then burns
iterations retrying URLs that can never succeed.

GOCLAW_SSRF_ALLOWED_CIDRS lets an operator name the ranges their proxy
hands out. Empty by default, so nothing changes for deployments that do
not set it, and the accepted and refused entries are both logged at
startup — this widens what LLM- and admin-supplied URLs can reach, so it
should be visible.

Ranges an SSRF actually targets can never be allowlisted: link-local
(including cloud metadata at 169.254.169.254), multicast and unspecified
are refused at parse time, in either direction, so neither an exact entry
nor a wider range that swallows one gets through.

Applied inside isBlocked rather than only in validate() because
NewSafeClient re-checks the pinned IP at dial time through the same
function — relaxing just the pre-flight check would pass validation and
then fail to connect.

internal/tools carries its own private-range list for web_fetch and
web_search, separate from this package and not identical to it. It has to
consult the same setting, or relaxing one gate leaves the other rejecting
the very traffic the operator permitted. Unifying the two lists is left
alone here; it is a wider change than this one.

Co-authored-by: Conner Mo <connermo@ConnerdeMacBook-Pro.local>

cmd/gateway_lifecycle.go

@@ -18,6 +18,7 @@ import (
 	"github.com/nextlevelbuilder/goclaw/internal/orchestration"
 	"github.com/nextlevelbuilder/goclaw/internal/sandbox"
 	"github.com/nextlevelbuilder/goclaw/internal/scheduler"
+	"github.com/nextlevelbuilder/goclaw/internal/security"
 	"github.com/nextlevelbuilder/goclaw/internal/store"
 	"github.com/nextlevelbuilder/goclaw/internal/tasks"
 	"github.com/nextlevelbuilder/goclaw/internal/tools"
@@ -401,6 +402,16 @@ func (d *gatewayDeps) runLifecycle(
 	} else if !edition.Current().IsLimited() {
 		slog.Warn("security.cors_open: no allowed_origins configured — all WebSocket origins accepted. Set gateway.allowed_origins or GOCLAW_ALLOWED_ORIGINS for production")
 	}
+	if allowed, rejected := security.OperatorAllowlistStatus(); len(allowed) > 0 || len(rejected) > 0 {
+		if len(allowed) > 0 {
+			slog.Warn("security.ssrf_allowlist: SSRF protection relaxed for operator-configured ranges — tool- and admin-supplied URLs may reach them",
+				"env", security.SSRFAllowedCIDRsEnv, "allowed", allowed)
+		}
+		if len(rejected) > 0 {
+			slog.Warn("security.ssrf_allowlist_rejected: entries refused; cloud-metadata, multicast and unspecified ranges can never be allowlisted",
+				"env", security.SSRFAllowedCIDRsEnv, "rejected", rejected)
+		}
+	}
 
 	if err := d.server.Start(ctx); err != nil {
 		slog.Error("gateway error", "error", err)

internal/security/ssrf.go

@@ -11,6 +11,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"os"
 	"strings"
 	"sync/atomic"
 	"time"
@@ -103,8 +104,144 @@ func isExemptable(ip net.IP) bool {
 	return false
 }
 
+// SSRFAllowedCIDRsEnv names the operator escape hatch for deployments where a
+// transparent proxy makes DNS resolution stop describing the real destination.
+const SSRFAllowedCIDRsEnv = "GOCLAW_SSRF_ALLOWED_CIDRS"
+
+// operatorAllowedCIDRs are ranges an operator has explicitly un-blocked via
+// GOCLAW_SSRF_ALLOWED_CIDRS (comma-separated). Empty by default, which leaves
+// the block list exactly as it ships.
+//
+// The case this exists for: a TUN/fake-IP proxy answers every DNS query with a
+// synthetic address out of a reserved range and then routes that address to the
+// real public host. The resolved IP is a handle, not a destination, so the
+// "resolve, then judge the IP" model this package is built on reports a false
+// positive on traffic that never touches an internal network. Without a way to
+// say so, web_fetch is unusable in that environment and no configuration can
+// fix it.
+//
+// This widens what LLM- and admin-supplied URLs can reach, so it is opt-in,
+// logged at startup, and refuses the ranges an SSRF actually targets.
+var operatorAllowedCIDRs []*net.IPNet
+
+// neverAllowlistableCIDRs can never be un-blocked, whatever the operator sets.
+// Cloud metadata (169.254.169.254) is the canonical SSRF target and must stay
+// unreachable; multicast and unspecified are meaningless as proxy handles.
+// This mirrors what exemptableCIDRs already refuses to cover.
+var neverAllowlistableCIDRs []*net.IPNet
+
+func init() {
+	for _, cidr := range []string{
+		"169.254.0.0/16", "fe80::/10", // link-local incl. cloud metadata
+		"224.0.0.0/4", "ff00::/8", // multicast
+		"0.0.0.0/32", "::/128", // unspecified
+	} {
+		_, ipNet, err := net.ParseCIDR(cidr)
+		if err != nil {
+			panic(fmt.Sprintf("security: bad never-allowlistable CIDR %q: %v", cidr, err))
+		}
+		neverAllowlistableCIDRs = append(neverAllowlistableCIDRs, ipNet)
+	}
+
+	nets, rejected := parseOperatorAllowedCIDRs(os.Getenv(SSRFAllowedCIDRsEnv))
+	operatorAllowedCIDRs = nets
+	operatorAllowlistRejected = rejected
+}
+
+// operatorAllowlistRejected records entries refused at parse time so startup
+// can report them. A silently dropped entry would read as "configured".
+var operatorAllowlistRejected []string
+
+// parseOperatorAllowedCIDRs parses a comma-separated CIDR list, dropping
+// entries that are malformed or that overlap a never-allowlistable range.
+// Returns the accepted nets and a human-readable reason per rejected entry.
+func parseOperatorAllowedCIDRs(spec string) (nets []*net.IPNet, rejected []string) {
+	for _, raw := range strings.Split(spec, ",") {
+		entry := strings.TrimSpace(raw)
+		if entry == "" {
+			continue
+		}
+		_, ipNet, err := net.ParseCIDR(entry)
+		if err != nil {
+			rejected = append(rejected, fmt.Sprintf("%s (not a CIDR)", entry))
+			continue
+		}
+		if blocked := overlappingNeverAllowlistable(ipNet); blocked != "" {
+			rejected = append(rejected, fmt.Sprintf("%s (overlaps %s)", entry, blocked))
+			continue
+		}
+		nets = append(nets, ipNet)
+	}
+	return nets, rejected
+}
+
+// overlappingNeverAllowlistable returns the never-allowlistable range that
+// candidate overlaps, or "" when it overlaps none. Both directions are checked
+// so neither a wider nor a narrower entry can slip a protected range through.
+func overlappingNeverAllowlistable(candidate *net.IPNet) string {
+	for _, protected := range neverAllowlistableCIDRs {
+		if candidate.Contains(protected.IP) || protected.Contains(candidate.IP) {
+			return protected.String()
+		}
+	}
+	return ""
+}
+
+// SetOperatorAllowlistForTest replaces the operator allowlist with the parsed
+// form of spec and returns a function restoring the previous value.
+//
+// This function MUST only be called from test code, and not from tests marked
+// t.Parallel(): unlike allowLoopbackForTest it swaps slices rather than an
+// atomic. It exists so packages that carry their own SSRF gate — internal/tools
+// — can cover the allowlist without duplicating the parser.
+func SetOperatorAllowlistForTest(spec string) func() {
+	prevNets, prevRejected := operatorAllowedCIDRs, operatorAllowlistRejected
+	operatorAllowedCIDRs, operatorAllowlistRejected = parseOperatorAllowedCIDRs(spec)
+	return func() {
+		operatorAllowedCIDRs, operatorAllowlistRejected = prevNets, prevRejected
+	}
+}
+
+// OperatorAllowlistStatus reports the configured allowlist and any rejected
+// entries, for the startup warning. Callers must not mutate the result.
+func OperatorAllowlistStatus() (allowed []string, rejected []string) {
+	for _, n := range operatorAllowedCIDRs {
+		allowed = append(allowed, n.String())
+	}
+	return allowed, operatorAllowlistRejected
+}
+
+// isOperatorAllowed reports whether an operator has un-blocked ip's range.
+func isOperatorAllowed(ip net.IP) bool {
+	for _, cidr := range operatorAllowedCIDRs {
+		if cidr.Contains(ip) {
+			return true
+		}
+	}
+	return false
+}
+
+// IsOperatorAllowed reports whether an operator has explicitly un-blocked ip's
+// range via GOCLAW_SSRF_ALLOWED_CIDRS.
+//
+// Exported for the separate SSRF check in internal/tools, which carries its own
+// private-range list. Both gates have to honour the same operator setting, or
+// relaxing one leaves the other rejecting the very traffic it was configured
+// to permit.
+func IsOperatorAllowed(ip net.IP) bool {
+	return isOperatorAllowed(ip)
+}
+
 // isBlocked returns true if ip falls within any blocked CIDR.
+//
+// The operator allowlist is consulted first and deliberately applies here
+// rather than only in validate(): NewSafeClient re-checks the pinned IP at dial
+// time through this same function, so relaxing only the pre-flight check would
+// pass validation and then fail to connect.
 func isBlocked(ip net.IP) bool {
+	if isOperatorAllowed(ip) {
+		return false
+	}
 	for _, cidr := range blockedCIDRs {
 		if cidr.Contains(ip) {
 			return true

internal/security/ssrf_operator_allowlist_test.go

@@ -0,0 +1,162 @@
+package security
+
+import (
+	"net"
+	"strings"
+	"testing"
+)
+
+// The escape hatch must be inert unless an operator opts in — an empty or
+// absent env var has to leave the shipped block list untouched.
+func TestParseOperatorAllowedCIDRs_EmptyIsInert(t *testing.T) {
+	t.Parallel()
+
+	for _, spec := range []string{"", "   ", ",", " , , "} {
+		nets, rejected := parseOperatorAllowedCIDRs(spec)
+		if len(nets) != 0 {
+			t.Fatalf("spec %q produced %d allowed nets, want 0", spec, len(nets))
+		}
+		if len(rejected) != 0 {
+			t.Fatalf("spec %q produced rejections %v, want none", spec, rejected)
+		}
+	}
+}
+
+// The case the hatch exists for: a TUN/fake-IP proxy hands out RFC 2544
+// addresses that route to real public hosts.
+func TestParseOperatorAllowedCIDRs_AcceptsBenchmarkingRange(t *testing.T) {
+	t.Parallel()
+
+	nets, rejected := parseOperatorAllowedCIDRs("198.18.0.0/15")
+	if len(rejected) != 0 {
+		t.Fatalf("unexpected rejections: %v", rejected)
+	}
+	if len(nets) != 1 {
+		t.Fatalf("got %d nets, want 1", len(nets))
+	}
+	if !nets[0].Contains(net.ParseIP("198.18.0.236")) {
+		t.Fatal("parsed net does not contain the address it was configured for")
+	}
+}
+
+// Cloud metadata is the canonical SSRF target. No operator setting may open a
+// path to it — not directly, and not by way of a wider range that swallows it.
+func TestParseOperatorAllowedCIDRs_RefusesProtectedRanges(t *testing.T) {
+	t.Parallel()
+
+	cases := []struct {
+		name string
+		spec string
+	}{
+		{"link-local exactly", "169.254.0.0/16"},
+		{"cloud metadata host", "169.254.169.254/32"},
+		{"a wider range swallowing link-local", "169.0.0.0/8"},
+		{"everything", "0.0.0.0/0"},
+		{"multicast", "224.0.0.0/4"},
+		{"unspecified", "0.0.0.0/32"},
+		{"ipv6 link-local", "fe80::/10"},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			nets, rejected := parseOperatorAllowedCIDRs(tc.spec)
+			if len(nets) != 0 {
+				t.Fatalf("%q was accepted; it must never be allowlistable", tc.spec)
+			}
+			if len(rejected) != 1 {
+				t.Fatalf("got %d rejections for %q, want 1", len(rejected), tc.spec)
+			}
+			if !strings.Contains(rejected[0], "overlaps") {
+				t.Fatalf("rejection %q does not explain the overlap", rejected[0])
+			}
+		})
+	}
+}
+
+// A rejected entry must be reported, never silently dropped — an operator who
+// mistyped one range out of several would otherwise believe it took effect.
+func TestParseOperatorAllowedCIDRs_ReportsBadEntriesAndKeepsGoodOnes(t *testing.T) {
+	t.Parallel()
+
+	nets, rejected := parseOperatorAllowedCIDRs("198.18.0.0/15, not-a-cidr, 169.254.0.0/16, 10.0.0.0/8")
+
+	if len(nets) != 2 {
+		t.Fatalf("got %d accepted nets, want 2 (198.18.0.0/15 and 10.0.0.0/8)", len(nets))
+	}
+	if len(rejected) != 2 {
+		t.Fatalf("got %d rejections, want 2; rejected=%v", len(rejected), rejected)
+	}
+
+	joined := strings.Join(rejected, " | ")
+	if !strings.Contains(joined, "not a CIDR") {
+		t.Fatalf("malformed entry not reported: %v", rejected)
+	}
+	if !strings.Contains(joined, "169.254.0.0/16") {
+		t.Fatalf("protected entry not reported: %v", rejected)
+	}
+}
+
+// isBlocked is the single gate both the pre-flight check and the dial-time
+// re-check go through, so the allowlist has to take effect inside it.
+func TestIsBlocked_HonorsOperatorAllowlist(t *testing.T) {
+	saved := operatorAllowedCIDRs
+	t.Cleanup(func() { operatorAllowedCIDRs = saved })
+
+	fakeIP := net.ParseIP("198.18.0.236")
+	metadata := net.ParseIP("169.254.169.254")
+	privateIP := net.ParseIP("10.1.2.3")
+
+	if !isBlocked(fakeIP) {
+		t.Fatal("198.18.0.236 should be blocked with no allowlist configured")
+	}
+
+	nets, _ := parseOperatorAllowedCIDRs("198.18.0.0/15")
+	operatorAllowedCIDRs = nets
+
+	if isBlocked(fakeIP) {
+		t.Fatal("198.18.0.236 should be permitted once its range is allowlisted")
+	}
+	if !isBlocked(metadata) {
+		t.Fatal("cloud metadata must stay blocked regardless of the allowlist")
+	}
+	if !isBlocked(privateIP) {
+		t.Fatal("RFC 1918 must stay blocked when only 198.18.0.0/15 is allowlisted")
+	}
+}
+
+// Validate is what web_fetch calls; it must agree with isBlocked so a URL that
+// passes validation can actually be dialed.
+func TestValidate_AllowsAllowlistedRangeEndToEnd(t *testing.T) {
+	saved := operatorAllowedCIDRs
+	t.Cleanup(func() { operatorAllowedCIDRs = saved })
+
+	nets, _ := parseOperatorAllowedCIDRs("198.18.0.0/15")
+	operatorAllowedCIDRs = nets
+
+	// Resolution is what makes this range interesting, so drive isBlocked with
+	// a literal-IP URL to keep the test off the network.
+	if _, _, err := validate("https://198.18.0.236/", false, nil); err != nil {
+		t.Fatalf("allowlisted address rejected by validate: %v", err)
+	}
+	if _, _, err := validate("https://169.254.169.254/", false, nil); err == nil {
+		t.Fatal("cloud metadata passed validate while an allowlist was configured")
+	}
+}
+
+func TestOperatorAllowlistStatus_ReportsConfiguredRanges(t *testing.T) {
+	savedNets, savedRejected := operatorAllowedCIDRs, operatorAllowlistRejected
+	t.Cleanup(func() {
+		operatorAllowedCIDRs, operatorAllowlistRejected = savedNets, savedRejected
+	})
+
+	operatorAllowedCIDRs, operatorAllowlistRejected = parseOperatorAllowedCIDRs("198.18.0.0/15, 169.254.0.0/16")
+
+	allowed, rejected := OperatorAllowlistStatus()
+	if len(allowed) != 1 || allowed[0] != "198.18.0.0/15" {
+		t.Fatalf("allowed = %v, want [198.18.0.0/15]", allowed)
+	}
+	if len(rejected) != 1 {
+		t.Fatalf("rejected = %v, want one entry", rejected)
+	}
+}

internal/tools/web_shared.go

@@ -8,6 +8,8 @@ import (
 	"sync"
 	"time"
 	"unicode/utf8"
+
+	"github.com/nextlevelbuilder/goclaw/internal/security"
 )
 
 // --- In-memory cache (matching TS src/agents/tools/web-shared.ts) ---
@@ -120,6 +122,16 @@ func isPrivateIP(ipStr string) bool {
 		return false
 	}
 
+	// An operator running behind a TUN/fake-IP proxy can un-block the synthetic
+	// range it hands out (GOCLAW_SSRF_ALLOWED_CIDRS). This check duplicates the
+	// range list in internal/security rather than sharing it, so it has to
+	// consult that setting explicitly — otherwise web_fetch keeps rejecting the
+	// addresses the operator just permitted. Cloud metadata can never be
+	// allowlisted, so this cannot open a path to it.
+	if security.IsOperatorAllowed(ip) {
+		return false
+	}
+
 	// IPv4 private ranges
 	privateRanges := []struct {
 		network string

internal/tools/web_shared_ssrf_allowlist_test.go

@@ -0,0 +1,46 @@
+package tools
+
+import (
+	"testing"
+
+	"github.com/nextlevelbuilder/goclaw/internal/security"
+)
+
+// withOperatorAllowlist installs an operator allowlist for the duration of t.
+func withOperatorAllowlist(t *testing.T, spec string) {
+	t.Helper()
+	t.Cleanup(security.SetOperatorAllowlistForTest(spec))
+}
+
+// internal/tools carries its own private-range list, separate from
+// internal/security. The operator allowlist has to reach both: relaxing only
+// the security package left web_fetch rejecting exactly the addresses the
+// operator had just permitted.
+func TestCheckSSRF_HonorsOperatorAllowlist(t *testing.T) {
+	// 198.18.0.0/15 is in the local list, so with no allowlist it must fail.
+	if err := CheckSSRF("https://198.18.0.236/"); err == nil {
+		t.Fatal("198.18.0.236 accepted with no allowlist configured")
+	}
+
+	withOperatorAllowlist(t, "198.18.0.0/15")
+
+	if err := CheckSSRF("https://198.18.0.236/"); err != nil {
+		t.Fatalf("allowlisted address still rejected by CheckSSRF: %v", err)
+	}
+}
+
+// Whatever the operator allowlists, the ranges an SSRF actually targets stay
+// unreachable.
+func TestCheckSSRF_KeepsProtectedRangesBlocked(t *testing.T) {
+	withOperatorAllowlist(t, "198.18.0.0/15")
+
+	for _, target := range []string{
+		"https://169.254.169.254/latest/meta-data/", // cloud metadata
+		"https://10.0.0.1/",                         // RFC 1918
+		"https://127.0.0.1/",                        // loopback
+	} {
+		if err := CheckSSRF(target); err == nil {
+			t.Fatalf("%s was accepted while only 198.18.0.0/15 was allowlisted", target)
+		}
+	}
+}