Merge remote-tracking branch 'origin/dev' into codex/issue-48-google-workspace-cli

# Conflicts:
#	docs/project-changelog.md
This commit is contained in:
Goon committed 2026-05-24 18:57:47 +07:00
commit 3a749ff91c
189 files changed
+8860 -848

No files matched your search

+187
View File
@@ -0,0 +1,187 @@
package tools
import (
"regexp"
"strconv"
"strings"
"github.com/nextlevelbuilder/goclaw/internal/config"
)
type commandKeywordAllowAudit struct {
RuleID string
Command string
Subcommand string
Arg string
Keyword string
Reason string
}
func applyCommandKeywordAllowlist(command string, args []string, rules []config.CommandKeywordAllowlistRule) ([]string, []commandKeywordAllowAudit) {
if len(args) == 0 || len(rules) == 0 {
return args, nil
}
var out []string
var audits []commandKeywordAllowAudit
for _, rule := range rules {
if !commandKeywordAllowlistRuleEnabled(rule) {
continue
}
if normalizeBinaryName(rule.Command) != normalizeBinaryName(command) {
continue
}
subcommand, argStart, ok := matchCommandKeywordSubcommand(args, rule.Subcommands)
if !ok {
continue
}
argNames := commandKeywordSet(rule.Args)
argPositions := commandKeywordPositionSet(rule.ArgPositions)
keywords := commandKeywordSet(rule.Keywords)
if (len(argNames) == 0 && len(argPositions) == 0) || len(keywords) == 0 {
continue
}
if len(argPositions) > 0 && subcommand == "" {
continue
}
if out == nil {
out = slicesClone(args)
}
for i := argStart; i < len(out); i++ {
argName, valueIndex, inlineValue, ok := commandKeywordArgValue(out, i, argNames)
if !ok {
if _, positional := argPositions[i-argStart]; positional {
argName = "argPositions[" + strconv.Itoa(i-argStart) + "]"
valueIndex = i
inlineValue = out[i]
ok = true
}
}
if !ok {
continue
}
value := inlineValue
if valueIndex != i {
value = out[valueIndex]
}
masked, hits := maskCommandKeywords(value, keywords)
if len(hits) == 0 {
continue
}
if valueIndex == i {
prefix := out[i][:strings.Index(out[i], "=")+1]
out[i] = prefix + masked
} else {
out[valueIndex] = masked
i = valueIndex
}
for _, keyword := range hits {
audits = append(audits, commandKeywordAllowAudit{
RuleID: rule.ID,
Command: normalizeBinaryName(command),
Subcommand: subcommand,
Arg: argName,
Keyword: keyword,
Reason: rule.Reason,
})
}
}
}
if out == nil {
return args, audits
}
return out, audits
}
func commandKeywordAllowlistRuleEnabled(rule config.CommandKeywordAllowlistRule) bool {
return rule.Enabled == nil || *rule.Enabled
}
func matchCommandKeywordSubcommand(args []string, subcommands []string) (string, int, bool) {
if len(subcommands) == 0 {
return "", 0, true
}
for _, subcommand := range subcommands {
parts := strings.Fields(strings.ToLower(strings.TrimSpace(subcommand)))
if len(parts) == 0 || len(parts) > len(args) {
continue
}
matched := true
for i, part := range parts {
if strings.ToLower(args[i]) != part {
matched = false
break
}
}
if matched {
return strings.Join(parts, " "), len(parts), true
}
}
return "", 0, false
}
func commandKeywordSet(values []string) map[string]struct{} {
if len(values) == 0 {
return nil
}
set := make(map[string]struct{}, len(values))
for _, value := range values {
value = strings.ToLower(strings.TrimSpace(value))
if value != "" {
set[value] = struct{}{}
}
}
return set
}
func commandKeywordPositionSet(values []int) map[int]struct{} {
if len(values) == 0 {
return nil
}
set := make(map[int]struct{}, len(values))
for _, value := range values {
if value >= 0 {
set[value] = struct{}{}
}
}
return set
}
func commandKeywordArgValue(args []string, i int, argNames map[string]struct{}) (string, int, string, bool) {
arg := args[i]
if eq := strings.Index(arg, "="); eq > 0 {
name := strings.ToLower(arg[:eq])
if _, ok := argNames[name]; ok {
return name, i, arg[eq+1:], true
}
return "", 0, "", false
}
name := strings.ToLower(arg)
if _, ok := argNames[name]; !ok || i+1 >= len(args) {
return "", 0, "", false
}
return name, i + 1, "", true
}
func maskCommandKeywords(value string, keywords map[string]struct{}) (string, []string) {
if value == "" || len(keywords) == 0 {
return value, nil
}
var hits []string
masked := value
for keyword := range keywords {
re := regexp.MustCompile(`(?i)\b` + regexp.QuoteMeta(keyword) + `\b`)
if !re.MatchString(masked) {
continue
}
hits = append(hits, keyword)
masked = re.ReplaceAllString(masked, "__allowlisted_keyword__")
}
return masked, hits
}
func slicesClone[T any](src []T) []T {
dst := make([]T, len(src))
copy(dst, src)
return dst
}
+24 -7
View File
@@ -39,8 +39,8 @@ var wrapperBinaries = map[string]bool{
// normalizeBinaryName returns the lowercased file base of a binary reference.
// Examples: "/usr/bin/gh" → "gh", "./GH" → "gh", " Gh " → "gh".
// Applied at BOTH the gate lookup and lookupCredentialedBinary so the two
// layers agree on identity. (Red Team F5)
// Applied at both the gate lookup and lookupCredentialedBinary so the two
// layers agree on identity.
func normalizeBinaryName(s string) string {
return filepath.Base(strings.TrimSpace(strings.ToLower(s)))
}
@@ -381,7 +381,8 @@ func (t *ExecTool) executeCredentialed(ctx context.Context, cred *store.SecureCL
}
// Step 3: Per-binary deny check (deny_args)
if p := matchesBinaryDeny(args, cred.DenyArgs); p != "" {
denyArgs, allowAudits := applyCommandKeywordAllowlist(binary, args, t.commandKeywordAllowlistSnapshot())
if p := matchesBinaryDeny(denyArgs, cred.DenyArgs); p != "" {
return credentialedDenyError(binary, args, p)
}
// Per-binary verbose deny check (deny_verbose) — per-arg start-anchored match
@@ -389,6 +390,20 @@ func (t *ExecTool) executeCredentialed(ctx context.Context, cred *store.SecureCL
if p := matchesBinaryVerbose(args, cred.DenyVerbose); p != "" {
return credentialedDenyError(binary, args, p)
}
for _, audit := range allowAudits {
slog.Info("security.command_keyword_allowlist",
"binary", audit.Command,
"subcommand", audit.Subcommand,
"arg", audit.Arg,
"keyword", audit.Keyword,
"rule_id", audit.RuleID,
"reason", audit.Reason,
"agent_id", store.AgentIDFromContext(ctx),
"user_id", store.UserIDFromContext(ctx),
"credential_user_id", store.CredentialUserIDFromContext(ctx),
"tenant_id", store.TenantIDFromContext(ctx),
)
}
// Step 4: Decrypt env vars from store (already decrypted by store layer).
// Per-user env overrides take priority over binary/grant env.
@@ -421,13 +436,15 @@ func mergeCredentialedEnv(cred *store.SecureCLIBinary) (map[string]string, error
return envMap, nil
}
if len(cred.EncryptedEnv) > 0 {
if err := json.Unmarshal(cred.EncryptedEnv, &envMap); err != nil {
baseEnv, err := store.FlattenSecureCLIEnv(cred.EncryptedEnv)
if err != nil {
return nil, err
}
maps.Copy(envMap, baseEnv)
}
if len(cred.UserEnv) > 0 {
var userEnvMap map[string]string
if err := json.Unmarshal(cred.UserEnv, &userEnvMap); err != nil {
userEnvMap, err := store.FlattenSecureCLIEnv(cred.UserEnv)
if err != nil {
return nil, err
}
maps.Copy(envMap, userEnvMap)
@@ -613,7 +630,7 @@ func (t *ExecTool) lookupCredentialedBinary(ctx context.Context, command string)
}
// Normalize lookup key so path/case variants (/usr/bin/gh, ./gh, GH) all
// resolve to the same registry row. Same helper is used by the gate
// branch in Execute — identity must agree at both layers. (Red Team F5)
// branch in Execute because identity must agree at both layers.
normBinary := normalizeBinaryName(binary)
// Get agent ID from context for scoped lookup
agentID := store.AgentIDFromContext(ctx)
@@ -43,3 +43,24 @@ func TestMergeCredentialedEnvFailsClosedOnInvalidUserEnv(t *testing.T) {
t.Fatal("expected invalid per-user env JSON to fail closed")
}
}
func TestMergeCredentialedEnvFlattensSensitiveValueEntries(t *testing.T) {
binary := &store.SecureCLIBinary{
EncryptedEnv: []byte(`{
"TOKEN":{"kind":"sensitive","value":"secret"},
"PUBLIC_BASE_URL":{"kind":"value","value":"https://goclaw.sh"}
}`),
UserEnv: []byte(`{"PUBLIC_BASE_URL":{"kind":"value","value":"https://user.example"}}`),
}
env, err := mergeCredentialedEnv(binary)
if err != nil {
t.Fatalf("mergeCredentialedEnv() error = %v", err)
}
if env["TOKEN"] != "secret" {
t.Fatalf("TOKEN = %q", env["TOKEN"])
}
if env["PUBLIC_BASE_URL"] != "https://user.example" {
t.Fatalf("PUBLIC_BASE_URL = %q", env["PUBLIC_BASE_URL"])
}
}
+109
View File
@@ -5,6 +5,8 @@ import (
"os"
"path/filepath"
"testing"
"github.com/nextlevelbuilder/goclaw/internal/config"
)
func TestDetectShellOperators(t *testing.T) {
@@ -208,6 +210,113 @@ func TestMatchesBinaryDenyJoinedArgs(t *testing.T) {
}
}
func TestApplyCommandKeywordAllowlistScopesContentArgs(t *testing.T) {
ghPatterns, _ := json.Marshal([]string{`auth\s+`, `repo\s+delete`, `secret\s+`, `token\s+`})
rules := []config.CommandKeywordAllowlistRule{
{
ID: "github-content",
Command: "gh",
Subcommands: []string{"issue create", "pr create"},
Args: []string{"--body", "--title"},
ArgPositions: []int{0},
Keywords: []string{"secret", "token"},
Reason: "GitHub issue and PR prose may discuss security terms.",
},
}
tests := []struct {
name string
args []string
wantHit bool
wantAudit int
}{
{
name: "issue body content allowed",
args: []string{"issue", "create", "--body", "secret rotation details"},
wantHit: false,
wantAudit: 1,
},
{
name: "pr title content allowed",
args: []string{"pr", "create", "--title=token handling notes"},
wantHit: false,
wantAudit: 1,
},
{
name: "positional content allowed",
args: []string{"issue", "create", "token handling notes"},
wantHit: false,
wantAudit: 1,
},
{
name: "command path stays blocked",
args: []string{"secret", "set", "TOKEN"},
wantHit: true,
},
{
name: "non-allowlisted arg stays blocked",
args: []string{"issue", "create", "--label", "secret incident"},
wantHit: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
sanitized, audits := applyCommandKeywordAllowlist("gh", tt.args, rules)
got := matchesBinaryDeny(sanitized, ghPatterns)
if (got != "") != tt.wantHit {
t.Fatalf("matchesBinaryDeny(%v) after allowlist = %q, wantHit=%v", sanitized, got, tt.wantHit)
}
if len(audits) != tt.wantAudit {
t.Fatalf("audit count = %d, want %d", len(audits), tt.wantAudit)
}
})
}
}
func TestApplyCommandKeywordAllowlistIgnoresDisabledRules(t *testing.T) {
ghPatterns, _ := json.Marshal([]string{`secret\s+`})
enabled := false
rules := []config.CommandKeywordAllowlistRule{
{
ID: "disabled-github-content",
Command: "gh",
Subcommands: []string{"issue create"},
Args: []string{"--body"},
Keywords: []string{"secret"},
Enabled: &enabled,
},
}
sanitized, audits := applyCommandKeywordAllowlist("gh", []string{"issue", "create", "--body", "secret notes"}, rules)
if got := matchesBinaryDeny(sanitized, ghPatterns); got == "" {
t.Fatalf("disabled rule bypassed deny_args; sanitized args = %v", sanitized)
}
if len(audits) != 0 {
t.Fatalf("disabled rule emitted audit records: %v", audits)
}
}
func TestApplyCommandKeywordAllowlistRequiresSubcommandForPositions(t *testing.T) {
ghPatterns, _ := json.Marshal([]string{`secret\s+`})
rules := []config.CommandKeywordAllowlistRule{
{
ID: "unsafe-position",
Command: "gh",
ArgPositions: []int{0},
Keywords: []string{"secret"},
},
}
sanitized, audits := applyCommandKeywordAllowlist("gh", []string{"secret", "set", "TOKEN"}, rules)
if got := matchesBinaryDeny(sanitized, ghPatterns); got == "" {
t.Fatalf("position rule without subcommand bypassed command-path deny; sanitized args = %v", sanitized)
}
if len(audits) != 0 {
t.Fatalf("position rule without subcommand emitted audit records: %v", audits)
}
}
func TestResolveAndMatchBinaryUsesConfiguredExecutablePath(t *testing.T) {
t.Setenv("PATH", "/usr/bin")
binDir := t.TempDir()
+6
View File
@@ -8,6 +8,7 @@ import (
"strings"
"github.com/nextlevelbuilder/goclaw/internal/providers"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
// --- Context helpers for media audio ---
@@ -45,12 +46,17 @@ var audioModelDefaults = map[string]string{
type ReadAudioTool struct {
registry *providers.Registry
mediaLoader MediaPathLoader
usageCaps *usagecaps.Service
}
func NewReadAudioTool(registry *providers.Registry, mediaLoader MediaPathLoader) *ReadAudioTool {
return &ReadAudioTool{registry: registry, mediaLoader: mediaLoader}
}
func (t *ReadAudioTool) SetUsageCapService(svc *usagecaps.Service) {
t.usageCaps = svc
}
func (t *ReadAudioTool) Name() string { return "read_audio" }
func (t *ReadAudioTool) Description() string {
+46 -2
View File
@@ -93,7 +93,19 @@ func (t *ReadAudioTool) callProvider(ctx context.Context, cp credentialProvider,
// providers exposing a /v1/audio/transcriptions endpoint.
if isTranscriptionModel(model) {
slog.Info("read_audio: using openai transcription API", "provider", providerName, "model", model, "size", len(data), "mime", mime)
chatReq := providers.ChatRequest{
Messages: []providers.Message{{Role: "user", Content: prompt}},
Model: model,
Options: map[string]any{"max_tokens": 16384},
}
reservation, reserveErr := reserveToolLLMUsage(ctx, t.usageCaps, t.Name(), providerName, model, chatReq)
if reserveErr != nil {
return nil, nil, reserveErr
}
resp, err := openaiTranscriptionCall(ctx, cp.APIKey(), cp.APIBase(), model, data, mime)
if reservation != nil {
reservation.Reconcile(ctx, resp, err)
}
if err != nil {
return nil, nil, fmt.Errorf("openai transcription call: %w", err)
}
@@ -103,7 +115,19 @@ func (t *ReadAudioTool) callProvider(ctx context.Context, cp credentialProvider,
// Gemini: use File API (inlineData doesn't work for audio).
if ptype == "gemini" {
slog.Info("read_audio: using gemini file API", "provider", providerName, "model", model, "size", len(data), "mime", mime)
chatReq := providers.ChatRequest{
Messages: []providers.Message{{Role: "user", Content: prompt}},
Model: model,
Options: map[string]any{"max_tokens": 16384},
}
reservation, reserveErr := reserveToolLLMUsage(ctx, t.usageCaps, t.Name(), providerName, model, chatReq)
if reserveErr != nil {
return nil, nil, reserveErr
}
resp, err := geminiFileAPICall(ctx, cp.APIKey(), model, prompt, data, mime, 120*time.Second)
if reservation != nil {
reservation.Reconcile(ctx, resp, err)
}
if err != nil {
return nil, nil, fmt.Errorf("gemini file API: %w", err)
}
@@ -113,7 +137,19 @@ func (t *ReadAudioTool) callProvider(ctx context.Context, cp credentialProvider,
// Native OpenAI chat-audio (gpt-4o-audio-preview etc.): input_audio content part.
if ptype == "openai" {
slog.Info("read_audio: using openai input_audio API", "provider", providerName, "model", model, "size", len(data), "mime", mime)
chatReq := providers.ChatRequest{
Messages: []providers.Message{{Role: "user", Content: prompt}},
Model: model,
Options: map[string]any{"max_tokens": 16384},
}
reservation, reserveErr := reserveToolLLMUsage(ctx, t.usageCaps, t.Name(), providerName, model, chatReq)
if reserveErr != nil {
return nil, nil, reserveErr
}
resp, err := openaiAudioCall(ctx, cp.APIKey(), cp.APIBase(), model, prompt, data, mime)
if reservation != nil {
reservation.Reconcile(ctx, resp, err)
}
if err != nil {
return nil, nil, fmt.Errorf("openai audio call: %w", err)
}
@@ -128,7 +164,7 @@ func (t *ReadAudioTool) callProvider(ctx context.Context, cp credentialProvider,
}
slog.Info("read_audio: using chat API fallback", "provider", providerName, "model", model, "size", len(data))
resp, err := p.Chat(ctx, providers.ChatRequest{
chatReq := providers.ChatRequest{
Messages: []providers.Message{
{
Role: "user",
@@ -141,7 +177,15 @@ func (t *ReadAudioTool) callProvider(ctx context.Context, cp credentialProvider,
"max_tokens": 16384,
"temperature": 0.2,
},
})
}
reservation, reserveErr := reserveToolLLMUsage(ctx, t.usageCaps, t.Name(), providerName, model, chatReq)
if reserveErr != nil {
return nil, nil, reserveErr
}
resp, err := p.Chat(ctx, chatReq)
if reservation != nil {
reservation.Reconcile(ctx, resp, err)
}
if err != nil {
return nil, nil, fmt.Errorf("chat API: %w", err)
}
+6
View File
@@ -8,6 +8,7 @@ import (
"strings"
"github.com/nextlevelbuilder/goclaw/internal/providers"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
// textReadableMIMEs are MIME types whose content can be returned directly without LLM analysis.
@@ -68,12 +69,17 @@ var documentModelDefaults = map[string]string{
type ReadDocumentTool struct {
registry *providers.Registry
mediaLoader MediaPathLoader
usageCaps *usagecaps.Service
}
func NewReadDocumentTool(registry *providers.Registry, mediaLoader MediaPathLoader) *ReadDocumentTool {
return &ReadDocumentTool{registry: registry, mediaLoader: mediaLoader}
}
func (t *ReadDocumentTool) SetUsageCapService(svc *usagecaps.Service) {
t.usageCaps = svc
}
func (t *ReadDocumentTool) Name() string { return "read_document" }
func (t *ReadDocumentTool) Description() string {
+27 -4
View File
@@ -43,8 +43,7 @@ func (t *ReadDocumentTool) resolveDocumentFile(ctx context.Context, mediaID, doc
}
}
if ref == nil {
slog.Warn("read_document: media_id not found, falling back to most recent", "media_id", mediaID)
ref = &refs[len(refs)-1]
return "", "", fmt.Errorf("document media_id %q not found in this conversation", mediaID)
}
} else {
// Use the last (most recent) document ref.
@@ -152,7 +151,23 @@ func (t *ReadDocumentTool) callProvider(ctx context.Context, cp credentialProvid
slog.Info("read_document: using gemini native API",
"provider", providerName, "model", model,
"doc_size", len(data), "mime", mime)
chatReq := providers.ChatRequest{
Messages: []providers.Message{{
Role: "user",
Content: prompt,
Images: []providers.ImageContent{{MimeType: mime}},
}},
Model: model,
Options: map[string]any{"max_tokens": 16384},
}
reservation, reserveErr := reserveToolLLMUsage(ctx, t.usageCaps, t.Name(), providerName, model, chatReq)
if reserveErr != nil {
return nil, nil, reserveErr
}
resp, err := geminiNativeDocumentCall(ctx, cp.APIKey(), model, prompt, data, mime)
if reservation != nil {
reservation.Reconcile(ctx, resp, err)
}
if err != nil {
return nil, nil, fmt.Errorf("gemini native call: %w", err)
}
@@ -179,7 +194,7 @@ func (t *ReadDocumentTool) callProvider(ctx context.Context, cp credentialProvid
opts["disable_tools"] = true
}
resp, err := p.Chat(ctx, providers.ChatRequest{
chatReq := providers.ChatRequest{
Messages: []providers.Message{
{
Role: "user",
@@ -189,7 +204,15 @@ func (t *ReadDocumentTool) callProvider(ctx context.Context, cp credentialProvid
},
Model: model,
Options: opts,
})
}
reservation, reserveErr := reserveToolLLMUsage(ctx, t.usageCaps, t.Name(), providerName, model, chatReq)
if reserveErr != nil {
return nil, nil, reserveErr
}
resp, err := p.Chat(ctx, chatReq)
if reservation != nil {
reservation.Reconcile(ctx, resp, err)
}
if err != nil {
return nil, nil, fmt.Errorf("chat call: %w", err)
}
+18 -1
View File
@@ -66,7 +66,7 @@ func TestResolveDocumentFileMatchesUploadedFilenameAlias(t *testing.T) {
}
}
func TestResolveDocumentFileInvalidMediaIDFallsBackToMostRecent(t *testing.T) {
func TestResolveDocumentFileInvalidMediaIDReturnsError(t *testing.T) {
refs := []providers.MediaRef{
{ID: uuid.NewString(), Kind: "document", Path: "/workspace/.uploads/old.pdf", MimeType: "application/pdf"},
{ID: uuid.NewString(), Kind: "document", Path: "/workspace/.uploads/latest.pdf", MimeType: "application/pdf"},
@@ -75,6 +75,23 @@ func TestResolveDocumentFileInvalidMediaIDFallsBackToMostRecent(t *testing.T) {
tool := NewReadDocumentTool(nil, nil)
ctx := WithMediaDocRefs(context.Background(), refs)
gotPath, _, err := tool.resolveDocumentFile(ctx, "not-a-real-media-id", "")
if err == nil {
t.Fatalf("resolveDocumentFile returned path %q, want explicit media_id error", gotPath)
}
if !strings.Contains(err.Error(), "not-a-real-media-id") {
t.Fatalf("error = %q, want requested media_id", err.Error())
}
}
func TestResolveDocumentFileOmittedMediaIDUsesLastRef(t *testing.T) {
refs := []providers.MediaRef{
{ID: uuid.NewString(), Kind: "document", Path: "/workspace/.uploads/old.pdf", MimeType: "application/pdf"},
{ID: uuid.NewString(), Kind: "document", Path: "/workspace/.uploads/latest.pdf", MimeType: "application/pdf"},
}
tool := NewReadDocumentTool(nil, nil)
ctx := WithMediaDocRefs(context.Background(), refs)
gotPath, _, err := tool.resolveDocumentFile(ctx, "", "")
if err != nil {
t.Fatalf("resolveDocumentFile returned error: %v", err)
}
+17 -3
View File
@@ -10,6 +10,7 @@ import (
"strings"
"github.com/nextlevelbuilder/goclaw/internal/providers"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
// --- Context helpers for media images ---
@@ -46,13 +47,18 @@ var visionModelDefaults = map[string]string{
// ReadImageTool uses a vision-capable provider to describe images attached to the current message.
type ReadImageTool struct {
registry *providers.Registry
registry *providers.Registry
usageCaps *usagecaps.Service
}
func NewReadImageTool(registry *providers.Registry) *ReadImageTool {
return &ReadImageTool{registry: registry}
}
func (t *ReadImageTool) SetUsageCapService(svc *usagecaps.Service) {
t.usageCaps = svc
}
func (t *ReadImageTool) Name() string { return "read_image" }
func (t *ReadImageTool) Description() string {
@@ -152,7 +158,7 @@ func (t *ReadImageTool) callProvider(ctx context.Context, cp credentialProvider,
opts["disable_tools"] = true
}
resp, err := p.Chat(ctx, providers.ChatRequest{
chatReq := providers.ChatRequest{
Messages: []providers.Message{
{
Role: "user",
@@ -162,7 +168,15 @@ func (t *ReadImageTool) callProvider(ctx context.Context, cp credentialProvider,
},
Model: model,
Options: opts,
})
}
reservation, reserveErr := reserveToolLLMUsage(ctx, t.usageCaps, t.Name(), providerName, model, chatReq)
if reserveErr != nil {
return nil, nil, reserveErr
}
resp, err := p.Chat(ctx, chatReq)
if reservation != nil {
reservation.Reconcile(ctx, resp, err)
}
if err != nil {
return nil, nil, fmt.Errorf("vision provider error: %w", err)
}
+6
View File
@@ -8,6 +8,7 @@ import (
"strings"
"github.com/nextlevelbuilder/goclaw/internal/providers"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
// --- Context helpers for media video ---
@@ -45,12 +46,17 @@ var videoModelDefaults = map[string]string{
type ReadVideoTool struct {
registry *providers.Registry
mediaLoader MediaPathLoader
usageCaps *usagecaps.Service
}
func NewReadVideoTool(registry *providers.Registry, mediaLoader MediaPathLoader) *ReadVideoTool {
return &ReadVideoTool{registry: registry, mediaLoader: mediaLoader}
}
func (t *ReadVideoTool) SetUsageCapService(svc *usagecaps.Service) {
t.usageCaps = svc
}
func (t *ReadVideoTool) Name() string { return "read_video" }
func (t *ReadVideoTool) Description() string {
+22 -2
View File
@@ -70,7 +70,19 @@ func (t *ReadVideoTool) callProvider(ctx context.Context, cp credentialProvider,
ptype := GetParamString(params, "_provider_type", providerTypeFromName(providerName))
if cp != nil && ptype == "gemini" {
slog.Info("read_video: using gemini file API", "provider", providerName, "model", model, "size", len(data), "mime", mime)
chatReq := providers.ChatRequest{
Messages: []providers.Message{{Role: "user", Content: prompt}},
Model: model,
Options: map[string]any{"max_tokens": 16384},
}
reservation, reserveErr := reserveToolLLMUsage(ctx, t.usageCaps, t.Name(), providerName, model, chatReq)
if reserveErr != nil {
return nil, nil, reserveErr
}
resp, err := geminiFileAPICall(ctx, cp.APIKey(), model, prompt, data, mime, 180*time.Second)
if reservation != nil {
reservation.Reconcile(ctx, resp, err)
}
if err != nil {
return nil, nil, fmt.Errorf("gemini file API: %w", err)
}
@@ -84,7 +96,7 @@ func (t *ReadVideoTool) callProvider(ctx context.Context, cp credentialProvider,
}
slog.Info("read_video: using chat API fallback", "provider", providerName, "model", model, "size", len(data))
resp, err := p.Chat(ctx, providers.ChatRequest{
chatReq := providers.ChatRequest{
Messages: []providers.Message{
{
Role: "user",
@@ -97,7 +109,15 @@ func (t *ReadVideoTool) callProvider(ctx context.Context, cp credentialProvider,
"max_tokens": 16384,
"temperature": 0.2,
},
})
}
reservation, reserveErr := reserveToolLLMUsage(ctx, t.usageCaps, t.Name(), providerName, model, chatReq)
if reserveErr != nil {
return nil, nil, reserveErr
}
resp, err := p.Chat(ctx, chatReq)
if reservation != nil {
reservation.Reconcile(ctx, resp, err)
}
if err != nil {
return nil, nil, fmt.Errorf("chat API: %w", err)
}
+51 -14
View File
@@ -13,8 +13,10 @@ import (
"regexp"
"runtime"
"strings"
"sync"
"time"
"github.com/nextlevelbuilder/goclaw/internal/config"
"github.com/nextlevelbuilder/goclaw/internal/sandbox"
"github.com/nextlevelbuilder/goclaw/internal/store"
"golang.org/x/text/unicode/norm"
@@ -35,27 +37,56 @@ func DefaultDenyPatterns() []*regexp.Regexp {
// ExecTool executes shell commands, optionally inside a sandbox container.
type ExecTool struct {
workspace string
timeout time.Duration
pathDenyPatterns []*regexp.Regexp // always-on path-based denials (DenyPaths)
pathDenyRoots []string // raw deny roots for nested workspace exemptions
denyExemptions []string // substrings that exempt a command from deny
restrict bool
sandboxMgr sandbox.Manager // nil = no sandbox, execute on host
approvalMgr *ExecApprovalManager // nil = no approval needed
agentID string // for approval request context
secureCLIStore store.SecureCLIStore // nil = no credentialed exec
workspace string
timeout time.Duration
pathDenyPatterns []*regexp.Regexp // always-on path-based denials (DenyPaths)
pathDenyRoots []string // raw deny roots for nested workspace exemptions
denyExemptions []string // substrings that exempt a command from deny
restrict bool
sandboxMgr sandbox.Manager // nil = no sandbox, execute on host
approvalMgr *ExecApprovalManager // nil = no approval needed
agentID string // for approval request context
secureCLIStore store.SecureCLIStore // nil = no credentialed exec
policyMu sync.RWMutex
commandKeywordAllowlist []config.CommandKeywordAllowlistRule
// globalDenyGroups holds global shell deny-group toggles from config.tools.
// Per-agent overrides from context (store.WithShellDenyGroups) win per-key.
// Updated at startup and via TopicConfigChanged pub/sub for runtime reload.
globalDenyGroups map[string]bool
}
// SetCommandKeywordAllowlist replaces the scoped credentialed CLI keyword
// allowlist. The slice is defensively copied so config reload callers cannot
// mutate the running tool policy after assignment.
func (t *ExecTool) SetCommandKeywordAllowlist(rules []config.CommandKeywordAllowlistRule) {
t.policyMu.Lock()
defer t.policyMu.Unlock()
if len(rules) == 0 {
t.commandKeywordAllowlist = nil
return
}
t.commandKeywordAllowlist = slicesClone(rules)
}
// CommandKeywordAllowlistForTest exposes the effective global allowlist for
// cross-package config reload tests. Not for production callers.
func (t *ExecTool) CommandKeywordAllowlistForTest() []config.CommandKeywordAllowlistRule {
return t.commandKeywordAllowlistSnapshot()
}
func (t *ExecTool) commandKeywordAllowlistSnapshot() []config.CommandKeywordAllowlistRule {
t.policyMu.RLock()
defer t.policyMu.RUnlock()
return slicesClone(t.commandKeywordAllowlist)
}
// SetGlobalShellDenyGroups replaces the global shell deny-group toggles. The
// caller's map is defensively copied so later mutations cannot leak into the
// tool's internal state. Passing nil or an empty map clears the global config
// (per-agent context overrides, if any, still apply on their own).
func (t *ExecTool) SetGlobalShellDenyGroups(groups map[string]bool) {
t.policyMu.Lock()
defer t.policyMu.Unlock()
if len(groups) == 0 {
t.globalDenyGroups = nil
return
@@ -70,14 +101,20 @@ func (t *ExecTool) SetGlobalShellDenyGroups(groups map[string]bool) {
// empty, the other is returned directly (no allocation).
func (t *ExecTool) effectiveDenyGroups(ctx context.Context) map[string]bool {
agent := store.ShellDenyGroupsFromContext(ctx)
if len(t.globalDenyGroups) == 0 {
t.policyMu.RLock()
global := t.globalDenyGroups
if len(global) > 0 {
global = maps.Clone(global)
}
t.policyMu.RUnlock()
if len(global) == 0 {
return agent
}
if len(agent) == 0 {
return t.globalDenyGroups
return global
}
merged := make(map[string]bool, len(t.globalDenyGroups)+len(agent))
maps.Copy(merged, t.globalDenyGroups)
merged := make(map[string]bool, len(global)+len(agent))
maps.Copy(merged, global)
// agent wins per-key
maps.Copy(merged, agent)
return merged
+36 -29
View File
@@ -18,6 +18,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/bus"
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/store"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
// SubagentConfig configures the subagent system.
@@ -40,33 +41,34 @@ const (
// SubagentTask tracks a running or completed subagent.
type SubagentTask struct {
ID string `json:"id"`
ParentID string `json:"parentId"`
Task string `json:"task"`
Label string `json:"label"`
Status string `json:"status"` // "running", "completed", "failed", "cancelled"
Result string `json:"result,omitempty"`
Depth int `json:"depth"`
Model string `json:"model,omitempty"` // model override for this subagent
TotalInputTokens int64 `json:"totalInputTokens,omitempty"`
TotalOutputTokens int64 `json:"totalOutputTokens,omitempty"`
OriginChannel string `json:"originChannel,omitempty"`
OriginChatID string `json:"originChatId,omitempty"`
OriginPeerKind string `json:"originPeerKind,omitempty"` // "direct" or "group" (for session key building)
OriginLocalKey string `json:"originLocalKey,omitempty"` // composite key with topic/thread suffix for routing
OriginUserID string `json:"originUserId,omitempty"` // parent's userID for per-user scoping propagation
OriginSenderID string `json:"originSenderId,omitempty"` // real acting sender; preserves permission attribution in announce re-ingress (#915)
OriginRole string `json:"originRole,omitempty"` // parent's RBAC role; bypasses per-user grants for admin/operator/owner in re-ingress (#915)
OriginSessionKey string `json:"originSessionKey,omitempty"` // exact parent session key for announce routing (WS uses non-standard format)
CreatedAt int64 `json:"createdAt"`
CompletedAt int64 `json:"completedAt,omitempty"`
Media []bus.MediaFile `json:"-"` // media files from tool results
OriginTenantID uuid.UUID `json:"-"` // parent's tenant for announce routing
OriginTraceID uuid.UUID `json:"-"` // parent trace for announce linking
OriginRootSpanID uuid.UUID `json:"-"` // parent agent's root span ID
cancelFunc context.CancelFunc `json:"-"` // per-task context cancel
spawnConfig SubagentConfig `json:"-"` // resolved config at spawn time (per-agent override merged)
dbID uuid.UUID `json:"-"` // persistent DB UUID (zero if not persisted)
ID string `json:"id"`
ParentID string `json:"parentId"`
Task string `json:"task"`
Label string `json:"label"`
Status string `json:"status"` // "running", "completed", "failed", "cancelled"
Result string `json:"result,omitempty"`
Depth int `json:"depth"`
Model string `json:"model,omitempty"` // model override for this subagent
TotalInputTokens int64 `json:"totalInputTokens,omitempty"`
TotalOutputTokens int64 `json:"totalOutputTokens,omitempty"`
OriginChannel string `json:"originChannel,omitempty"`
OriginChatID string `json:"originChatId,omitempty"`
OriginPeerKind string `json:"originPeerKind,omitempty"` // "direct" or "group" (for session key building)
OriginLocalKey string `json:"originLocalKey,omitempty"` // composite key with topic/thread suffix for routing
OriginUserID string `json:"originUserId,omitempty"` // parent's userID for per-user scoping propagation
OriginSenderID string `json:"originSenderId,omitempty"` // real acting sender; preserves permission attribution in announce re-ingress (#915)
OriginRole string `json:"originRole,omitempty"` // parent's RBAC role; bypasses per-user grants for admin/operator/owner in re-ingress (#915)
OriginSessionKey string `json:"originSessionKey,omitempty"` // exact parent session key for announce routing (WS uses non-standard format)
CreatedAt int64 `json:"createdAt"`
CompletedAt int64 `json:"completedAt,omitempty"`
Media []bus.MediaFile `json:"-"` // media files from tool results
OriginAgentID uuid.UUID `json:"-"` // parent agent UUID for usage caps and scoped tools
OriginTenantID uuid.UUID `json:"-"` // parent's tenant for announce routing
OriginTraceID uuid.UUID `json:"-"` // parent trace for announce linking
OriginRootSpanID uuid.UUID `json:"-"` // parent agent's root span ID
cancelFunc context.CancelFunc `json:"-"` // per-task context cancel
spawnConfig SubagentConfig `json:"-"` // resolved config at spawn time (per-agent override merged)
dbID uuid.UUID `json:"-"` // persistent DB UUID (zero if not persisted)
}
// SubagentManager manages the lifecycle of spawned subagents.
@@ -74,8 +76,8 @@ type SubagentManager struct {
mu sync.RWMutex
tasks map[string]*SubagentTask
config SubagentConfig
provider providers.Provider // default provider (fallback)
providerReg *providers.Registry // registry for resolving parent's provider
provider providers.Provider // default provider (fallback)
providerReg *providers.Registry // registry for resolving parent's provider
model string
msgBus *bus.MessageBus
@@ -83,6 +85,7 @@ type SubagentManager struct {
createTools func() *Registry
announceQueue *AnnounceQueue // optional: batches announces with debounce
taskStore store.SubagentTaskStore // optional: persists tasks to DB (fire-and-forget)
usageCaps *usagecaps.Service
}
// NewSubagentManager creates a new subagent manager.
@@ -116,6 +119,10 @@ func (sm *SubagentManager) SetTaskStore(s store.SubagentTaskStore) {
sm.taskStore = s
}
func (sm *SubagentManager) SetUsageCapService(s *usagecaps.Service) {
sm.usageCaps = s
}
// effectiveConfig returns the per-agent context override merged with defaults,
// or falls back to sm.config when no override is present.
func (sm *SubagentManager) effectiveConfig(ctx context.Context) SubagentConfig {
+43 -4
View File
@@ -13,6 +13,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/store"
"github.com/nextlevelbuilder/goclaw/internal/tracing"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
// runTask executes the subagent in a goroutine.
@@ -253,10 +254,7 @@ func (sm *SubagentManager) executeTask(ctx context.Context, task *SubagentTask)
}
slog.Info("subagent LLM retry", "id", task.ID, "iteration", iteration, "attempt", attempt+1)
}
// ctx is the parent agent's run context — cancelling the parent (e.g. agent abort)
// cascades here and to all subsequent tool calls in this iteration.
// Do NOT replace ctx with context.Background() here; that would detach abort propagation.
resp, err = activeProvider.Chat(ctx, chatReq)
resp, err = sm.chatSubagentWithUsageCap(ctx, task, activeProvider, model, chatReq, iteration, attempt+1)
if err == nil {
break
}
@@ -344,3 +342,44 @@ func (sm *SubagentManager) executeTask(ctx context.Context, task *SubagentTask)
return iteration
}
func (sm *SubagentManager) chatSubagentWithUsageCap(ctx context.Context, task *SubagentTask, activeProvider providers.Provider, model string, chatReq providers.ChatRequest, iteration, attempt int) (*providers.ChatResponse, error) {
if fallbackProvider, ok := activeProvider.(*providers.ModelFallbackProvider); ok {
before := func(callCtx context.Context, entry providers.FallbackCandidate, actualReq providers.ChatRequest) (providers.FallbackAfterCall, error) {
reservation, err := sm.reserveSubagentUsage(callCtx, task, entry.ProviderName, actualReq.Model, actualReq, fmt.Sprintf("%d:%d:%s:%s", iteration, attempt, entry.ProviderName, actualReq.Model))
if err != nil {
return nil, err
}
return func(resp *providers.ChatResponse, callErr error, _ providers.FallbackCallInfo) {
if reservation != nil {
reservation.Reconcile(callCtx, resp, callErr)
}
}, nil
}
return fallbackProvider.ChatWithHook(ctx, chatReq, before)
}
reservation, err := sm.reserveSubagentUsage(ctx, task, activeProvider.Name(), model, chatReq, fmt.Sprintf("%d:%d", iteration, attempt))
if err != nil {
return nil, err
}
resp, err := activeProvider.Chat(ctx, chatReq)
if reservation != nil {
reservation.Reconcile(ctx, resp, err)
}
return resp, err
}
func (sm *SubagentManager) reserveSubagentUsage(ctx context.Context, task *SubagentTask, providerName, model string, chatReq providers.ChatRequest, suffix string) (*usagecaps.Reservation, error) {
if sm.usageCaps == nil {
return nil, nil
}
return sm.usageCaps.Preflight(ctx, usagecaps.Request{
TenantID: task.OriginTenantID,
AgentID: task.OriginAgentID,
ProviderName: providerName,
ModelID: model,
ReservationKey: fmt.Sprintf("%s:%s", task.ID, suffix),
Messages: chatReq.Messages,
MaxOutputTokens: 4096,
})
}
+12 -10
View File
@@ -82,16 +82,17 @@ func (sm *SubagentManager) Spawn(
OriginChannel: channel,
OriginChatID: chatID,
OriginPeerKind: peerKind,
OriginLocalKey: ToolLocalKeyFromCtx(ctx),
OriginUserID: store.UserIDFromContext(ctx),
OriginSenderID: store.SenderIDFromContext(ctx),
OriginRole: store.RoleFromContext(ctx),
OriginSessionKey: ToolSessionKeyFromCtx(ctx),
OriginTenantID: store.TenantIDFromContext(ctx),
OriginTraceID: tracing.TraceIDFromContext(ctx),
OriginRootSpanID: tracing.ParentSpanIDFromContext(ctx),
CreatedAt: time.Now().UnixMilli(),
spawnConfig: cfg,
OriginLocalKey: ToolLocalKeyFromCtx(ctx),
OriginUserID: store.UserIDFromContext(ctx),
OriginSenderID: store.SenderIDFromContext(ctx),
OriginRole: store.RoleFromContext(ctx),
OriginSessionKey: ToolSessionKeyFromCtx(ctx),
OriginAgentID: store.AgentIDFromContext(ctx),
OriginTenantID: store.TenantIDFromContext(ctx),
OriginTraceID: tracing.TraceIDFromContext(ctx),
OriginRootSpanID: tracing.ParentSpanIDFromContext(ctx),
CreatedAt: time.Now().UnixMilli(),
spawnConfig: cfg,
}
// Detach from parent's cancellation chain so subagent survives after parent run completes.
// WithoutCancel preserves all context values (agent ID, workspace, trace info, etc.)
@@ -166,6 +167,7 @@ func (sm *SubagentManager) RunSync(
OriginSenderID: store.SenderIDFromContext(ctx),
OriginRole: store.RoleFromContext(ctx),
OriginSessionKey: ToolSessionKeyFromCtx(ctx),
OriginAgentID: store.AgentIDFromContext(ctx),
OriginTenantID: store.TenantIDFromContext(ctx),
OriginTraceID: tracing.TraceIDFromContext(ctx),
OriginRootSpanID: tracing.ParentSpanIDFromContext(ctx),
+44
View File
@@ -0,0 +1,44 @@
package tools
import (
"context"
"fmt"
"github.com/google/uuid"
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/store"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
func reserveToolLLMUsage(ctx context.Context, svc *usagecaps.Service, toolName, providerName, model string, req providers.ChatRequest) (*usagecaps.Reservation, error) {
if svc == nil {
return nil, nil
}
return svc.Preflight(ctx, usagecaps.Request{
TenantID: store.TenantIDFromContext(ctx),
AgentID: store.AgentIDFromContext(ctx),
ProviderName: providerName,
ModelID: model,
ReservationKey: fmt.Sprintf("tool:%s:%s", toolName, uuid.NewString()),
Messages: req.Messages,
MaxOutputTokens: maxOutputTokensFromOptions(req.Options),
})
}
func maxOutputTokensFromOptions(options map[string]any) int {
maxTokens := 1024
if options == nil {
return maxTokens
}
if v, ok := options[providers.OptMaxTokens]; ok {
switch n := v.(type) {
case int:
maxTokens = n
case int64:
maxTokens = int(n)
case float64:
maxTokens = int(n)
}
}
return maxTokens
}