mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-11 03:13:24 +00:00
fix(collaboration): harden delegated task isolation (#1486)
* feat(collaboration): isolate delegated artifacts and child runs Isolate delegated inputs and outputs behind secure artifact exchange lifecycles. Scope Agent Link tasks by tenant and root agent, and enforce delegation spawn-tree boundaries. Add process-wide child-run admission and preserve logical media paths across native, MCP, and sandbox execution. * fix(collaboration): harden delegated task isolation Enforce tenant and root-agent task scope across migrations and stores. Add exactly-once async completion delivery, delegated sandbox boundaries, and confined artifact and media recovery across runtime surfaces. * fix(collaboration): recover interrupted async tasks * fix(collaboration): normalize persisted child-run status --------- Co-authored-by: ntduc <ntduc@cpp.ai.vn>
This commit is contained in:
1 parent
496b7ffce6
commit
bb7712a9ff
186 files changed
+18858
-1233
No files matched your search
@@ -112,6 +112,7 @@ goclaw-patched-linux-amd64
|
||||
ui/web/nginx.staging.conf
|
||||
|
||||
# Local scratchpad / one-shot artifacts (never commit)
|
||||
/.artifacts/
|
||||
**/debug-*.log
|
||||
/*.sql
|
||||
scripts/docker-patch-*.sql
|
||||
@@ -252,7 +252,11 @@ Open **About** dialog → click **Update Now** (admin only). The update includes
|
||||
<img src="_statics/Agent Orchestration.jpg" alt="Agent Orchestration" width="800" />
|
||||
</p>
|
||||
|
||||
Each agent runs with its own identity, tools, LLM provider, and context files. Three delegation modes — sync (wait), async (fire-and-forget), bidirectional — connected through explicit permission links with concurrency limits.
|
||||
Each agent runs with its own identity, tools, LLM provider, and context files.
|
||||
Agent Links define outbound, inbound, or bidirectional permission edges.
|
||||
Delegation can run synchronously or asynchronously and exchanges files through
|
||||
an isolated delegation workspace; validated outputs are published back under
|
||||
the caller's `.delegations/<delegation-id>/` directory.
|
||||
|
||||
> Details: [Agent Teams docs](https://docs.goclaw.sh/#teams-what-are-teams)
|
||||
|
||||
|
||||
+47
-2
@@ -43,6 +43,7 @@ import (
|
||||
mcpbridge "github.com/nextlevelbuilder/goclaw/internal/mcp"
|
||||
mcpoauth "github.com/nextlevelbuilder/goclaw/internal/mcp/oauth"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/media"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/orchestration"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/providers"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/scheduler"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/security"
|
||||
@@ -98,6 +99,29 @@ type snapshotBucketRefresher interface {
|
||||
RefreshBuckets(context.Context, []time.Time) (int, error)
|
||||
}
|
||||
|
||||
func recoverInterruptedSubagentTasks(
|
||||
ctx context.Context,
|
||||
stores *store.Stores,
|
||||
retryDelay time.Duration,
|
||||
) (int64, error) {
|
||||
recoveryCtx := store.WithTenantID(ctx, store.MasterTenantID)
|
||||
for {
|
||||
recovered, err := stores.SubagentTaskRecovery.RecoverInterrupted(recoveryCtx)
|
||||
if err == nil {
|
||||
return recovered, nil
|
||||
}
|
||||
slog.Warn("subagent_tasks.recover_interrupted_retrying", "err", err)
|
||||
|
||||
timer := time.NewTimer(retryDelay)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return 0, ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func backfillTraceCostsAfterPricingSync(ctx context.Context, stores *store.Stores, snapshots snapshotBucketRefresher) {
|
||||
if stores == nil {
|
||||
return
|
||||
@@ -292,6 +316,24 @@ func runGateway() {
|
||||
}
|
||||
}
|
||||
|
||||
// Accepted async child runs are process-owned and cannot resume after a
|
||||
// restart. Reconcile their durable rows before wiring tools or accepting
|
||||
// traffic so completion lookups never remain queued/running forever.
|
||||
if pgStores.SubagentTaskRecovery != nil {
|
||||
startupCtx, stopStartup := signal.NotifyContext(
|
||||
context.Background(), syscall.SIGINT, syscall.SIGTERM,
|
||||
)
|
||||
n, err := recoverInterruptedSubagentTasks(startupCtx, pgStores, time.Second)
|
||||
stopStartup()
|
||||
if err != nil {
|
||||
slog.Info("subagent_tasks.recover_interrupted_aborted", "err", err)
|
||||
return
|
||||
}
|
||||
if n > 0 {
|
||||
slog.Info("subagent_tasks.recover_interrupted", "count", n)
|
||||
}
|
||||
}
|
||||
|
||||
if traceCollector != nil {
|
||||
defer traceCollector.Stop()
|
||||
// OTel OTLP export: compiled via build tags. Build with 'go build -tags otel' to enable.
|
||||
@@ -456,7 +498,8 @@ func runGateway() {
|
||||
}
|
||||
|
||||
// Subagent system (secureCLI store wired so subagent ExecTools enforce the gate)
|
||||
subagentMgr := setupSubagents(providerRegistry, cfg, msgBus, toolsReg, workspace, sandboxMgr, pgStores.SecureCLI, usageCapSvc)
|
||||
childRunAdmission := orchestration.NewChildRunAdmission(edition.Current().ChildRunLimit(), 128)
|
||||
subagentMgr := setupSubagents(providerRegistry, cfg, msgBus, toolsReg, workspace, sandboxMgr, pgStores.SecureCLI, usageCapSvc, childRunAdmission)
|
||||
if subagentMgr != nil {
|
||||
// Wire announce queue for batched subagent result delivery (matching TS debounce pattern).
|
||||
announceQueue := tools.NewAnnounceQueue(1000, 20, makeDelegateAnnounceCallback(subagentMgr, msgBus))
|
||||
@@ -577,7 +620,7 @@ func runGateway() {
|
||||
var mcpPool *mcpbridge.Pool
|
||||
var mediaStore *media.Store
|
||||
var postTurn tools.PostTurnProcessor
|
||||
contextFileInterceptor, mcpPool, mediaStore, postTurn = wireExtras(pgStores, agentRouter, providerRegistry, modelReg, msgBus, pgStores.Sessions, toolsReg, toolPE, skillsLoader, hasMemory, traceCollector, workspace, cfg.Gateway.InjectionAction, cfg, sandboxMgr, redisClient, domainBus, usageCapSvc, mcpOAuthRefresher)
|
||||
contextFileInterceptor, mcpPool, mediaStore, postTurn = wireExtras(pgStores, agentRouter, providerRegistry, modelReg, msgBus, pgStores.Sessions, toolsReg, toolPE, skillsLoader, hasMemory, traceCollector, workspace, cfg.Gateway.InjectionAction, cfg, sandboxMgr, redisClient, domainBus, usageCapSvc, mcpOAuthRefresher, childRunAdmission)
|
||||
if mcpPool != nil {
|
||||
defer mcpPool.Stop()
|
||||
}
|
||||
@@ -1042,9 +1085,11 @@ func runGateway() {
|
||||
sandboxMgr: sandboxMgr,
|
||||
postTurn: postTurn,
|
||||
subagentMgr: subagentMgr,
|
||||
childRunAdmission: childRunAdmission,
|
||||
consumerTeamStore: consumerTeamStore,
|
||||
auditCh: auditCh,
|
||||
sigCh: sigCh,
|
||||
terminateProcess: os.Exit,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/nextlevelbuilder/goclaw/internal/bus"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/config"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/memory"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/orchestration"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/providers"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/sandbox"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
@@ -163,7 +164,7 @@ func buildEmbeddingProvider(
|
||||
return nil
|
||||
}
|
||||
|
||||
func setupSubagents(providerReg *providers.Registry, cfg *config.Config, msgBus *bus.MessageBus, toolsReg *tools.Registry, workspace string, sandboxMgr sandbox.Manager, secureCLIStore store.SecureCLIStore, usageCapSvc *usagecaps.Service) *tools.SubagentManager {
|
||||
func setupSubagents(providerReg *providers.Registry, cfg *config.Config, msgBus *bus.MessageBus, toolsReg *tools.Registry, workspace string, sandboxMgr sandbox.Manager, secureCLIStore store.SecureCLIStore, usageCapSvc *usagecaps.Service, admission *orchestration.ChildRunAdmission) *tools.SubagentManager {
|
||||
names := providerReg.List(context.Background())
|
||||
if len(names) == 0 {
|
||||
return nil
|
||||
@@ -211,7 +212,7 @@ func setupSubagents(providerReg *providers.Registry, cfg *config.Config, msgBus
|
||||
return reg
|
||||
}
|
||||
|
||||
manager := tools.NewSubagentManager(provider, providerReg, agentCfg.Model, msgBus, toolsFactory, subCfg)
|
||||
manager := tools.NewSubagentManagerWithAdmission(provider, providerReg, agentCfg.Model, msgBus, toolsFactory, subCfg, admission)
|
||||
manager.SetUsageCapService(usageCapSvc)
|
||||
manager.SetAgentBudget(agentCfg.ContextWindow, agentCfg.MaxTokens)
|
||||
return manager
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/orchestration"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
)
|
||||
|
||||
func TestDrainChildRunsWithRetryReturnsTypedFailure(t *testing.T) {
|
||||
admission := orchestration.NewChildRunAdmission(1, 2)
|
||||
started := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
ticket, err := admission.Enqueue(context.Background(), orchestration.ChildRunConstraints{
|
||||
TenantID: store.MasterTenantID,
|
||||
TaskID: uuid.NewString(),
|
||||
}, func(context.Context, *orchestration.ChildRunLease) {
|
||||
close(started)
|
||||
<-release
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ticket.Activate(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("child run did not start")
|
||||
}
|
||||
|
||||
err = drainChildRunsWithRetry(admission, 10*time.Millisecond, 10*time.Millisecond)
|
||||
if !errors.Is(err, orchestration.ErrChildRunDrainTimeout) {
|
||||
t.Fatalf("drain error = %v, want typed timeout", err)
|
||||
}
|
||||
|
||||
close(release)
|
||||
select {
|
||||
case <-ticket.Done():
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("child run did not finish after forced termination signal")
|
||||
}
|
||||
if err := admission.Close(context.Background()); err != nil {
|
||||
t.Fatalf("final drain: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -79,6 +79,7 @@ func handleSubagentAnnounce(
|
||||
if sid := msg.Metadata[tools.MetaOriginRootSpanID]; sid != "" {
|
||||
parentRootSpanID, _ = uuid.Parse(sid)
|
||||
}
|
||||
rootAgentID, _ := uuid.Parse(msg.Metadata[tools.MetaSubagentRootAgentID])
|
||||
|
||||
// Group-scoped UserID for subagent announce (same logic as main lane).
|
||||
announceUserID := msg.UserID
|
||||
@@ -116,9 +117,7 @@ func handleSubagentAnnounce(
|
||||
originSenderID := msg.Metadata[tools.MetaOriginSenderID]
|
||||
originRole := msg.Metadata[tools.MetaOriginRole]
|
||||
|
||||
queueKey := fmt.Sprintf("%s:%s", msg.TenantID, sessionKey)
|
||||
routing := subagentAnnounceRouting{
|
||||
QueueKey: queueKey,
|
||||
SessionKey: sessionKey,
|
||||
TenantID: msg.TenantID,
|
||||
OrigChannel: origChannel,
|
||||
@@ -130,10 +129,16 @@ func handleSubagentAnnounce(
|
||||
SenderID: originSenderID,
|
||||
Role: originRole,
|
||||
ParentAgent: parentAgent,
|
||||
RootAgentID: rootAgentID,
|
||||
ParentTraceID: parentTraceID,
|
||||
ParentRootSpanID: parentRootSpanID,
|
||||
OutMeta: buildAnnounceOutMeta(origLocalKey),
|
||||
}
|
||||
// Batch only announces with identical routing and authority. A session can
|
||||
// be shared by multiple group senders or local topic keys; using only
|
||||
// tenant+session would let the first item's sender/role govern the rest.
|
||||
queueKey := subagentAnnounceRoutingKey(routing)
|
||||
routing.QueueKey = queueKey
|
||||
|
||||
// Enqueue into producer-consumer queue using tenant-scoped key from routing.
|
||||
isProcessor := enqueueSubagentAnnounce(queueKey, entry)
|
||||
@@ -142,7 +147,9 @@ func handleSubagentAnnounce(
|
||||
defer safego.Recover(nil, "component", "subagent_announce_loop", "session", sessionKey)
|
||||
|
||||
// Fetch live roster for merged announce context.
|
||||
roster := deps.SubagentMgr.RosterForParent(parentAgent)
|
||||
roster := deps.SubagentMgr.RosterForParent(tools.TaskScope{
|
||||
TenantID: msg.TenantID, RootAgentID: rootAgentID, RootAgentKey: parentAgent,
|
||||
})
|
||||
|
||||
processSubagentAnnounceLoop(ctx, routing, roster, deps.SubagentMgr, deps.Sched, deps.MsgBus, deps.Cfg, deps.ChannelMgr)
|
||||
})
|
||||
@@ -151,6 +158,22 @@ func handleSubagentAnnounce(
|
||||
return true
|
||||
}
|
||||
|
||||
func subagentAnnounceRoutingKey(r subagentAnnounceRouting) string {
|
||||
return strings.Join([]string{
|
||||
r.TenantID.String(),
|
||||
r.RootAgentID.String(),
|
||||
r.ParentAgent,
|
||||
r.SessionKey,
|
||||
r.OrigChannel,
|
||||
r.OrigChatID,
|
||||
r.OrigPeerKind,
|
||||
r.OrigLocalKey,
|
||||
r.UserID,
|
||||
r.SenderID,
|
||||
r.Role,
|
||||
}, "\x00")
|
||||
}
|
||||
|
||||
// handleTeammateMessage processes teammate messages: bypass debounce, route to target
|
||||
// agent session using the "team" lane, then announce result back to lead.
|
||||
// Returns true if the message was handled (caller should continue).
|
||||
@@ -560,11 +583,14 @@ func buildTeammateAnnounce(ctx context.Context, outcome scheduler.RunOutcome, se
|
||||
} else if outcome.Result == nil {
|
||||
slog.Warn("teammate message: nil result without error", "from", senderID)
|
||||
return "", nil, false
|
||||
} else if (outcome.Result.Content == "" && len(outcome.Result.Media) == 0) || agent.IsSilentReply(outcome.Result.Content) {
|
||||
} else if normalized, shouldDeliver := normalizeAgentOutboundContent(
|
||||
outcome.Result.Content,
|
||||
len(outcome.Result.Media),
|
||||
); !shouldDeliver {
|
||||
slog.Info("teammate message: suppressed silent/empty reply", "from", senderID)
|
||||
return "", nil, false
|
||||
} else {
|
||||
content = outcome.Result.Content
|
||||
content = normalized
|
||||
media = outcome.Result.Media
|
||||
}
|
||||
|
||||
|
||||
@@ -575,9 +575,14 @@ func processNormalMessage(
|
||||
return
|
||||
}
|
||||
|
||||
// Suppress empty/NO_REPLY responses (matching TS normalize-reply.ts).
|
||||
// Still publish an empty outbound so channels can clean up placeholder/thinking indicators.
|
||||
if outcome.Result.Content == "" || agent.IsSilentReply(outcome.Result.Content) {
|
||||
// Suppress silent text only when the result has no media. A tool or
|
||||
// delegate may legitimately return NO_REPLY with an attached artifact;
|
||||
// that must continue as a media-only outbound message.
|
||||
resultContent, shouldDeliver := normalizeAgentOutboundContent(
|
||||
outcome.Result.Content,
|
||||
len(outcome.Result.Media),
|
||||
)
|
||||
if !shouldDeliver {
|
||||
slog.Info("inbound: suppressed silent/empty reply",
|
||||
"channel", channel,
|
||||
"chat_id", chatID,
|
||||
@@ -615,7 +620,7 @@ func processNormalMessage(
|
||||
// Sanitize voice agent replies: replace technical errors with user-friendly fallback.
|
||||
replyContent := voiceguard.SanitizeReply(
|
||||
deps.Cfg.Channels.Telegram.VoiceAgentID, agentKey,
|
||||
channel, peerKind, inboundContent, outcome.Result.Content,
|
||||
channel, peerKind, inboundContent, resultContent,
|
||||
deps.Cfg.Channels.Telegram.AudioGuardFallbackTranscript,
|
||||
deps.Cfg.Channels.Telegram.AudioGuardFallbackNoTranscript,
|
||||
deps.Cfg.Channels.Telegram.AudioGuardErrorMarkers,
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
package cmd
|
||||
|
||||
import "github.com/nextlevelbuilder/goclaw/internal/agent"
|
||||
|
||||
// normalizeAgentOutboundContent keeps silent text out of user-facing messages
|
||||
// while allowing attached media to continue through channel delivery.
|
||||
func normalizeAgentOutboundContent(content string, mediaCount int) (string, bool) {
|
||||
isSilent := content == "" || agent.IsSilentReply(content)
|
||||
if !isSilent {
|
||||
return content, true
|
||||
}
|
||||
if mediaCount == 0 {
|
||||
return "", false
|
||||
}
|
||||
return "", true
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/agent"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/bus"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/sandbox"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/tools"
|
||||
)
|
||||
|
||||
// buildAgentLinkRunRequest preserves the origin's authorization-bearing
|
||||
// identity while keeping delegation on its internal delivery channel.
|
||||
func buildAgentLinkRunRequest(req tools.DelegateRequest, sessionKey string) agent.RunRequest {
|
||||
return agent.RunRequest{
|
||||
RunID: uuid.New().String(),
|
||||
SessionKey: sessionKey,
|
||||
Message: req.Task,
|
||||
UserID: req.UserID,
|
||||
SenderID: req.SenderID,
|
||||
Role: req.Role,
|
||||
Channel: "delegate",
|
||||
ChannelType: req.ChannelType,
|
||||
ChatID: req.ChatID,
|
||||
PeerKind: req.PeerKind,
|
||||
RunKind: "delegate",
|
||||
DelegationID: req.DelegationID,
|
||||
ParentAgentID: req.FromAgentKey,
|
||||
WorkspaceChannel: req.Channel,
|
||||
WorkspaceChatID: req.ChatID,
|
||||
DelegateInputsPath: req.DelegateInputsPath,
|
||||
DelegateOutputsPath: req.DelegateOutputsPath,
|
||||
}
|
||||
}
|
||||
|
||||
// Agent Link artifacts are published from the exchange manifest. Raw delegate
|
||||
// media paths point into B's ephemeral workspace and must never cross back to A.
|
||||
func agentMediaToBusFiles(_ []agent.MediaResult) []bus.MediaFile { return nil }
|
||||
|
||||
func releaseDelegationSandbox(ctx context.Context, manager sandbox.Manager, sessionKey string) error {
|
||||
if manager == nil {
|
||||
return nil
|
||||
}
|
||||
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
if err := manager.Release(cleanupCtx, sessionKey); err != nil {
|
||||
return fmt.Errorf("delegation sandbox release failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/agent"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/sandbox"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/tools"
|
||||
)
|
||||
|
||||
type delegationReleaseManager struct {
|
||||
released string
|
||||
err error
|
||||
}
|
||||
|
||||
func (m *delegationReleaseManager) Get(context.Context, string, string, *sandbox.Config, ...sandbox.GetOption) (sandbox.Sandbox, error) {
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
func (m *delegationReleaseManager) Release(_ context.Context, key string) error {
|
||||
m.released = key
|
||||
return m.err
|
||||
}
|
||||
func (*delegationReleaseManager) ReleaseAll(context.Context) error { return nil }
|
||||
func (*delegationReleaseManager) Stop() {}
|
||||
func (*delegationReleaseManager) Stats() map[string]any { return nil }
|
||||
|
||||
func TestBuildAgentLinkRunRequestPreservesGroupAuthorizationScope(t *testing.T) {
|
||||
req := tools.DelegateRequest{
|
||||
FromAgentKey: "coordinator",
|
||||
Task: "create the report",
|
||||
DelegationID: uuid.NewString(),
|
||||
UserID: "group:telegram:-100123",
|
||||
SenderID: "386246614",
|
||||
Role: "viewer",
|
||||
Channel: "telegram-main",
|
||||
ChannelType: "telegram",
|
||||
ChatID: "-100123",
|
||||
PeerKind: "group",
|
||||
DelegateInputsPath: filepath.Join(t.TempDir(), "inputs"),
|
||||
DelegateOutputsPath: filepath.Join(t.TempDir(), "outputs"),
|
||||
}
|
||||
|
||||
got := buildAgentLinkRunRequest(req, "delegate:session")
|
||||
|
||||
if got.UserID != req.UserID || got.SenderID != req.SenderID || got.Role != req.Role {
|
||||
t.Fatalf("authorization scope = (%q, %q, %q), want (%q, %q, %q)",
|
||||
got.UserID, got.SenderID, got.Role, req.UserID, req.SenderID, req.Role)
|
||||
}
|
||||
if got.Channel != "delegate" || got.ChannelType != req.ChannelType {
|
||||
t.Fatalf("channel = (%q, %q), want (delegate, %q)", got.Channel, got.ChannelType, req.ChannelType)
|
||||
}
|
||||
if got.ChatID != req.ChatID || got.PeerKind != req.PeerKind {
|
||||
t.Fatalf("chat scope = (%q, %q), want (%q, %q)",
|
||||
got.ChatID, got.PeerKind, req.ChatID, req.PeerKind)
|
||||
}
|
||||
if got.WorkspaceChannel != req.Channel || got.WorkspaceChatID != req.ChatID {
|
||||
t.Fatalf("workspace scope = (%q, %q), want (%q, %q)",
|
||||
got.WorkspaceChannel, got.WorkspaceChatID, req.Channel, req.ChatID)
|
||||
}
|
||||
if got.RunID == "" || got.RunKind != "delegate" || got.DelegationID != req.DelegationID {
|
||||
t.Fatalf("delegate classification = %#v", got)
|
||||
}
|
||||
if got.DelegateInputsPath != req.DelegateInputsPath ||
|
||||
got.DelegateOutputsPath != req.DelegateOutputsPath {
|
||||
t.Fatalf("artifact runtime wiring = (%q, %q), want (%q, %q)",
|
||||
got.DelegateInputsPath, got.DelegateOutputsPath,
|
||||
req.DelegateInputsPath, req.DelegateOutputsPath)
|
||||
}
|
||||
if got.Media != nil {
|
||||
t.Fatalf("run media = %#v, want exchange-only input delivery", got.Media)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentMediaToBusFilesDiscardsEphemeralDelegateMedia(t *testing.T) {
|
||||
got := agentMediaToBusFiles([]agent.MediaResult{{
|
||||
Path: filepath.Join(t.TempDir(), "ephemeral.png"),
|
||||
ContentType: "image/png",
|
||||
}})
|
||||
if got != nil {
|
||||
t.Fatalf("media = %#v, want raw delegate media discarded", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReleaseDelegationSandboxUsesExactSessionKey(t *testing.T) {
|
||||
manager := &delegationReleaseManager{}
|
||||
const sessionKey = "delegate:from:target:4fe8220f-07f1-4e64-a95c-b49ebc39db4a"
|
||||
if err := releaseDelegationSandbox(context.Background(), manager, sessionKey); err != nil {
|
||||
t.Fatalf("releaseDelegationSandbox: %v", err)
|
||||
}
|
||||
if manager.released != sessionKey {
|
||||
t.Fatalf("released key = %q, want %q", manager.released, sessionKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReleaseDelegationSandboxRedactsManagerFailure(t *testing.T) {
|
||||
manager := &delegationReleaseManager{err: errors.New("/private/exchange/inputs remained mounted")}
|
||||
err := releaseDelegationSandbox(context.Background(), manager, "delegate:session")
|
||||
if err == nil {
|
||||
t.Fatal("releaseDelegationSandbox unexpectedly succeeded")
|
||||
}
|
||||
if got := err.Error(); got != "delegation sandbox release failed" {
|
||||
t.Fatalf("error = %q, want redacted release failure", got)
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
@@ -14,6 +15,7 @@ import (
|
||||
"github.com/nextlevelbuilder/goclaw/internal/config"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/edition"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/heartbeat"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/orchestration"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/sandbox"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/scheduler"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
@@ -33,9 +35,71 @@ type lifecycleDeps struct {
|
||||
sandboxMgr sandbox.Manager
|
||||
postTurn tools.PostTurnProcessor
|
||||
subagentMgr *tools.SubagentManager
|
||||
childRunAdmission *orchestration.ChildRunAdmission
|
||||
consumerTeamStore store.TeamStore
|
||||
auditCh chan bus.AuditEventPayload
|
||||
sigCh chan os.Signal
|
||||
terminateProcess func(int)
|
||||
}
|
||||
|
||||
func drainChildRunsWithRetry(
|
||||
admission *orchestration.ChildRunAdmission,
|
||||
firstTimeout time.Duration,
|
||||
retryTimeout time.Duration,
|
||||
) error {
|
||||
if admission == nil {
|
||||
return nil
|
||||
}
|
||||
for attempt, timeout := range []time.Duration{firstTimeout, retryTimeout} {
|
||||
drainCtx, drainCancel := context.WithTimeout(context.Background(), timeout)
|
||||
err := admission.Close(drainCtx)
|
||||
drainCancel()
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
slog.Error("gateway: child-run drain attempt failed",
|
||||
"attempt", attempt+1, "timeout", timeout, "error", err)
|
||||
}
|
||||
return fmt.Errorf("%w after retry", orchestration.ErrChildRunDrainTimeout)
|
||||
}
|
||||
|
||||
func drainSubagentManagerWithRetry(
|
||||
manager *tools.SubagentManager,
|
||||
firstTimeout time.Duration,
|
||||
retryTimeout time.Duration,
|
||||
) error {
|
||||
if manager == nil {
|
||||
return nil
|
||||
}
|
||||
for attempt, timeout := range []time.Duration{firstTimeout, retryTimeout} {
|
||||
drainCtx, drainCancel := context.WithTimeout(context.Background(), timeout)
|
||||
err := manager.CloseContext(drainCtx)
|
||||
drainCancel()
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
slog.Error("gateway: subagent lifecycle drain attempt failed",
|
||||
"attempt", attempt+1, "timeout", timeout, "error", err)
|
||||
}
|
||||
return fmt.Errorf("%w after retry", tools.ErrSubagentLifecycleDrainTimeout)
|
||||
}
|
||||
|
||||
func drainDelegateToolWithRetry(
|
||||
tool interface{ CloseContext(context.Context) error },
|
||||
firstTimeout time.Duration,
|
||||
retryTimeout time.Duration,
|
||||
) error {
|
||||
for attempt, timeout := range []time.Duration{firstTimeout, retryTimeout} {
|
||||
drainCtx, drainCancel := context.WithTimeout(context.Background(), timeout)
|
||||
err := tool.CloseContext(drainCtx)
|
||||
drainCancel()
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
slog.Error("gateway: delegate completion drain attempt failed",
|
||||
"attempt", attempt+1, "timeout", timeout, "error", err)
|
||||
}
|
||||
return fmt.Errorf("delegate completion drain failed after retry")
|
||||
}
|
||||
|
||||
// runLifecycle wires config-reload subscribers, starts consumers, task recovery,
|
||||
@@ -198,6 +262,20 @@ func (d *gatewayDeps) runLifecycle(
|
||||
// Broadcast shutdown event
|
||||
d.server.BroadcastEvent(*protocol.NewEvent(protocol.EventShutdown, nil))
|
||||
|
||||
// Close child-run intake first. A drain timeout must terminate without
|
||||
// unwinding runGateway defers under a still-live child callback.
|
||||
if deps.childRunAdmission != nil {
|
||||
if err := drainChildRunsWithRetry(deps.childRunAdmission, 30*time.Second, 5*time.Second); err != nil {
|
||||
slog.Error("gateway: terminating after child-run drain failure", "error", err)
|
||||
terminate := deps.terminateProcess
|
||||
if terminate == nil {
|
||||
terminate = os.Exit
|
||||
}
|
||||
terminate(1)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Stop channels, cron, heartbeat, and task ticker
|
||||
d.channelMgr.StopAll(context.Background())
|
||||
d.pgStores.Cron.Stop()
|
||||
@@ -216,6 +294,35 @@ func (d *gatewayDeps) runLifecycle(
|
||||
close(deps.auditCh)
|
||||
}
|
||||
|
||||
if delegate, ok := d.toolsReg.Get("delegate"); ok {
|
||||
if closer, ok := delegate.(interface {
|
||||
CloseContext(context.Context) error
|
||||
}); ok {
|
||||
if err := drainDelegateToolWithRetry(closer, 65*time.Second, 10*time.Second); err != nil {
|
||||
slog.Error("gateway: terminating after delegate completion drain failure", "error", err)
|
||||
terminate := deps.terminateProcess
|
||||
if terminate == nil {
|
||||
terminate = os.Exit
|
||||
}
|
||||
terminate(1)
|
||||
return
|
||||
}
|
||||
} else if closer, ok := delegate.(interface{ Close() }); ok {
|
||||
closer.Close()
|
||||
}
|
||||
}
|
||||
if deps.subagentMgr != nil {
|
||||
if err := drainSubagentManagerWithRetry(deps.subagentMgr, 65*time.Second, 10*time.Second); err != nil {
|
||||
slog.Error("gateway: terminating after subagent lifecycle drain failure", "error", err)
|
||||
terminate := deps.terminateProcess
|
||||
if terminate == nil {
|
||||
terminate = os.Exit
|
||||
}
|
||||
terminate(1)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Close provider resources (e.g. Claude CLI temp files)
|
||||
d.providerRegistry.Close()
|
||||
|
||||
|
||||
+23
-15
@@ -62,6 +62,7 @@ func wireExtras(
|
||||
domainBus eventbus.DomainEventBus,
|
||||
usageCapSvc *usagecaps.Service,
|
||||
mcpOAuthProvider mcpbridge.OAuthTokenProvider, // nil = OAuth injection disabled
|
||||
childRunAdmission *orchestration.ChildRunAdmission,
|
||||
) (*tools.ContextFileInterceptor, *mcpbridge.Pool, *media.Store, tools.PostTurnProcessor) {
|
||||
// 1. Build cache instances (in-memory or Redis depending on build tags)
|
||||
agentCtxCache, userCtxCache := makeCaches(redisClient)
|
||||
@@ -450,25 +451,32 @@ func wireExtras(
|
||||
// Link delegate trace to parent trace
|
||||
delegateCtx := tracing.WithDelegateParentTraceID(ctx, tracing.TraceIDFromContext(ctx))
|
||||
|
||||
runReq := agent.RunRequest{
|
||||
RunID: uuid.New().String(),
|
||||
SessionKey: sessionKey,
|
||||
Message: req.Task,
|
||||
UserID: req.UserID,
|
||||
Channel: "delegate",
|
||||
RunKind: "delegate",
|
||||
DelegationID: req.DelegationID,
|
||||
ParentAgentID: req.FromAgentKey,
|
||||
runReq := buildAgentLinkRunRequest(req, sessionKey)
|
||||
var delegateTraceID uuid.UUID
|
||||
runReq.OnTraceCreated = func(traceID uuid.UUID) {
|
||||
delegateTraceID = traceID
|
||||
if req.OnTraceCreated != nil {
|
||||
req.OnTraceCreated(traceID)
|
||||
}
|
||||
}
|
||||
result, err := loop.Run(delegateCtx, runReq)
|
||||
if err != nil {
|
||||
return tools.DelegateResult{}, err
|
||||
result, runErr := loop.Run(delegateCtx, runReq)
|
||||
if releaseErr := releaseDelegationSandbox(ctx, sandboxMgr, sessionKey); releaseErr != nil {
|
||||
return tools.DelegateResult{TraceID: delegateTraceID}, releaseErr
|
||||
}
|
||||
cr := orchestration.CaptureFromRunResult(result, 0)
|
||||
return tools.DelegateResult{Content: cr.Content, Media: cr.Media}, nil
|
||||
if runErr != nil {
|
||||
return tools.DelegateResult{TraceID: delegateTraceID}, runErr
|
||||
}
|
||||
return tools.DelegateResult{
|
||||
Content: result.Content,
|
||||
Media: agentMediaToBusFiles(result.Media),
|
||||
TraceID: delegateTraceID,
|
||||
}, nil
|
||||
}
|
||||
delegateTool := tools.NewDelegateTool(stores.AgentLinks, stores.Agents, domainBus, delegateRunFn)
|
||||
delegateTool := tools.NewDelegateToolWithAdmission(stores.AgentLinks, stores.Agents, domainBus, delegateRunFn, childRunAdmission)
|
||||
delegateTool.SetDataDir(appCfg.DataDir)
|
||||
delegateTool.SetWorkspace(workspace)
|
||||
delegateTool.SetMsgBus(msgBus)
|
||||
delegateTool.SetTaskStore(stores.SubagentTasks)
|
||||
delegateTool.SetHookDispatcher(hookDispatcher)
|
||||
toolsReg.Register(delegateTool)
|
||||
slog.Info("delegate tool wired")
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func TestSubagentAnnounceRoutingKeyScopesRoutingAndAuthority(t *testing.T) {
|
||||
base := subagentAnnounceRouting{
|
||||
TenantID: uuid.New(),
|
||||
RootAgentID: uuid.New(),
|
||||
ParentAgent: "root",
|
||||
SessionKey: "session",
|
||||
OrigChannel: "telegram",
|
||||
OrigChatID: "chat",
|
||||
OrigPeerKind: "group",
|
||||
OrigLocalKey: "topic",
|
||||
UserID: "user",
|
||||
SenderID: "sender",
|
||||
Role: "operator",
|
||||
}
|
||||
want := subagentAnnounceRoutingKey(base)
|
||||
|
||||
cases := map[string]func(*subagentAnnounceRouting){
|
||||
"tenant": func(v *subagentAnnounceRouting) { v.TenantID = uuid.New() },
|
||||
"root id": func(v *subagentAnnounceRouting) { v.RootAgentID = uuid.New() },
|
||||
"parent agent": func(v *subagentAnnounceRouting) { v.ParentAgent += "-other" },
|
||||
"session": func(v *subagentAnnounceRouting) { v.SessionKey += "-other" },
|
||||
"channel": func(v *subagentAnnounceRouting) { v.OrigChannel += "-other" },
|
||||
"chat": func(v *subagentAnnounceRouting) { v.OrigChatID += "-other" },
|
||||
"peer kind": func(v *subagentAnnounceRouting) { v.OrigPeerKind += "-other" },
|
||||
"local key": func(v *subagentAnnounceRouting) { v.OrigLocalKey += "-other" },
|
||||
"user": func(v *subagentAnnounceRouting) { v.UserID += "-other" },
|
||||
"sender": func(v *subagentAnnounceRouting) { v.SenderID += "-other" },
|
||||
"role": func(v *subagentAnnounceRouting) { v.Role += "-other" },
|
||||
}
|
||||
for name, mutate := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
got := base
|
||||
mutate(&got)
|
||||
if key := subagentAnnounceRoutingKey(got); key == want {
|
||||
t.Fatalf("routing key did not change when %s changed", name)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -27,7 +27,9 @@ func makeDelegateAnnounceCallback(
|
||||
msgBus *bus.MessageBus,
|
||||
) func(sessionKey string, items []tools.AnnounceQueueItem, meta tools.AnnounceMetadata) {
|
||||
return func(sessionKey string, items []tools.AnnounceQueueItem, meta tools.AnnounceMetadata) {
|
||||
roster := subagentMgr.RosterForParent(meta.ParentAgent)
|
||||
roster := subagentMgr.RosterForParent(tools.TaskScope{
|
||||
TenantID: meta.OriginTenantID, RootAgentID: meta.RootAgentID, RootAgentKey: meta.ParentAgent,
|
||||
})
|
||||
content := tools.FormatBatchedAnnounce(items, roster)
|
||||
senderID := fmt.Sprintf("subagent:batch-%d", len(items))
|
||||
label := items[0].Label
|
||||
@@ -35,12 +37,13 @@ func makeDelegateAnnounceCallback(
|
||||
label = fmt.Sprintf("%d tasks", len(items))
|
||||
}
|
||||
batchMeta := map[string]string{
|
||||
tools.MetaOriginChannel: meta.OriginChannel,
|
||||
tools.MetaOriginPeerKind: meta.OriginPeerKind,
|
||||
tools.MetaParentAgent: meta.ParentAgent,
|
||||
tools.MetaSubagentLabel: label,
|
||||
tools.MetaOriginTraceID: meta.OriginTraceID,
|
||||
tools.MetaOriginRootSpanID: meta.OriginRootSpanID,
|
||||
tools.MetaOriginChannel: meta.OriginChannel,
|
||||
tools.MetaOriginPeerKind: meta.OriginPeerKind,
|
||||
tools.MetaParentAgent: meta.ParentAgent,
|
||||
tools.MetaSubagentRootAgentID: meta.RootAgentID.String(),
|
||||
tools.MetaSubagentLabel: label,
|
||||
tools.MetaOriginTraceID: meta.OriginTraceID,
|
||||
tools.MetaOriginRootSpanID: meta.OriginRootSpanID,
|
||||
}
|
||||
if meta.OriginLocalKey != "" {
|
||||
batchMeta[tools.MetaOriginLocalKey] = meta.OriginLocalKey
|
||||
@@ -69,7 +72,7 @@ func makeDelegateAnnounceCallback(
|
||||
"tasks": len(items),
|
||||
})
|
||||
|
||||
msgBus.PublishInbound(bus.InboundMessage{
|
||||
delivered := tools.PublishAsyncCompletion(context.Background(), msgBus, bus.InboundMessage{
|
||||
Channel: "system",
|
||||
SenderID: senderID,
|
||||
ChatID: meta.OriginChatID,
|
||||
@@ -79,6 +82,30 @@ func makeDelegateAnnounceCallback(
|
||||
Metadata: batchMeta,
|
||||
Media: batchMedia,
|
||||
})
|
||||
for _, item := range items {
|
||||
if !item.DurablyPersisted {
|
||||
slog.Error("subagent.batch_announce_without_durable_terminal",
|
||||
"task_id", item.SubagentID,
|
||||
"completion_id", item.CompletionID,
|
||||
"root_agent_id", meta.RootAgentID,
|
||||
"delivered", delivered,
|
||||
)
|
||||
continue
|
||||
}
|
||||
subagentMgr.UpdateAnnouncementStatus(
|
||||
store.WithTenantID(context.Background(), meta.OriginTenantID),
|
||||
meta.RootAgentID,
|
||||
item.CompletionID,
|
||||
delivered,
|
||||
)
|
||||
}
|
||||
if !delivered {
|
||||
slog.Warn("subagent.batch_announce_deferred_to_ledger",
|
||||
"root_agent_id", meta.RootAgentID,
|
||||
"batch_size", len(items),
|
||||
"reason", "inbound_bus_full",
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -96,7 +123,7 @@ type subagentAnnounceEntry struct {
|
||||
|
||||
// subagentAnnounceRouting holds shared routing info captured by the first enqueue.
|
||||
type subagentAnnounceRouting struct {
|
||||
QueueKey string // tenant-scoped key for sync.Map (tenantID:sessionKey)
|
||||
QueueKey string // tenant/root/session/topic/user/authority-scoped key
|
||||
SessionKey string // original session key (no tenant prefix) for RunRequest
|
||||
TenantID uuid.UUID // preserved for tenant-scoped scheduling
|
||||
OrigChannel string
|
||||
@@ -108,6 +135,7 @@ type subagentAnnounceRouting struct {
|
||||
SenderID string // real acting sender (preserves permission attribution through re-ingress, #915)
|
||||
Role string // caller's RBAC role; bypasses per-user grants for admin/operator/owner (#915)
|
||||
ParentAgent string
|
||||
RootAgentID uuid.UUID
|
||||
ParentTraceID uuid.UUID
|
||||
ParentRootSpanID uuid.UUID
|
||||
OutMeta map[string]string
|
||||
@@ -156,7 +184,9 @@ func processSubagentAnnounceLoop(
|
||||
}
|
||||
|
||||
// Refresh roster each iteration for up-to-date task statuses.
|
||||
roster = subagentMgr.RosterForParent(r.ParentAgent)
|
||||
roster = subagentMgr.RosterForParent(tools.TaskScope{
|
||||
TenantID: r.TenantID, RootAgentID: r.RootAgentID, RootAgentKey: r.ParentAgent,
|
||||
})
|
||||
content := buildMergedSubagentAnnounce(entries, roster)
|
||||
|
||||
// Collect media from all entries.
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/bus"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/tools"
|
||||
)
|
||||
|
||||
type recordingGatewayTaskStore struct {
|
||||
metadata chan map[string]any
|
||||
}
|
||||
|
||||
func (*recordingGatewayTaskStore) Create(context.Context, *store.SubagentTaskData) error {
|
||||
return nil
|
||||
}
|
||||
func (*recordingGatewayTaskStore) Get(context.Context, uuid.UUID, uuid.UUID) (*store.SubagentTaskData, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (*recordingGatewayTaskStore) UpdateStatus(context.Context, uuid.UUID, uuid.UUID, string, *string, int, int64, int64) error {
|
||||
return nil
|
||||
}
|
||||
func (*recordingGatewayTaskStore) ListByParent(context.Context, uuid.UUID, string) ([]store.SubagentTaskData, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (*recordingGatewayTaskStore) ListBySession(context.Context, uuid.UUID, string) ([]store.SubagentTaskData, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (*recordingGatewayTaskStore) Archive(context.Context, uuid.UUID, time.Duration, int) (int64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
func (s *recordingGatewayTaskStore) UpdateMetadata(_ context.Context, _ uuid.UUID, _ uuid.UUID, metadata map[string]any) error {
|
||||
s.metadata <- metadata
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestSubagentBatchAnnouncementDoesNotBlockOnFullInboundBus(t *testing.T) {
|
||||
messageBus := bus.New()
|
||||
for range 1000 {
|
||||
messageBus.PublishInbound(bus.InboundMessage{Content: "fill"})
|
||||
}
|
||||
manager := tools.NewSubagentManager(nil, nil, "", messageBus, nil, tools.SubagentConfig{})
|
||||
taskStore := &recordingGatewayTaskStore{metadata: make(chan map[string]any, 1)}
|
||||
manager.SetTaskStore(taskStore)
|
||||
callback := makeDelegateAnnounceCallback(manager, messageBus)
|
||||
completionID := uuid.New()
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
callback(
|
||||
"session-1",
|
||||
[]tools.AnnounceQueueItem{{
|
||||
SubagentID: "task-1",
|
||||
CompletionID: completionID,
|
||||
DurablyPersisted: true,
|
||||
Label: "probe",
|
||||
Status: tools.TaskStatusCompleted,
|
||||
Result: "done",
|
||||
}},
|
||||
tools.AnnounceMetadata{
|
||||
OriginChatID: "chat-1",
|
||||
OriginSessionKey: "session-1",
|
||||
OriginTenantID: uuid.New(),
|
||||
RootAgentID: uuid.New(),
|
||||
ParentAgent: "root",
|
||||
},
|
||||
)
|
||||
close(done)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("batched subagent announcement blocked on a full inbound bus")
|
||||
}
|
||||
select {
|
||||
case metadata := <-taskStore.metadata:
|
||||
if metadata["announcement_status"] != "undelivered" {
|
||||
t.Fatalf("announcement metadata = %#v, want undelivered", metadata)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("batched missed announcement was not recorded after bus saturation")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
)
|
||||
|
||||
type scriptedSubagentTaskRecovery struct {
|
||||
mu sync.Mutex
|
||||
failures int
|
||||
recovered int64
|
||||
calls int
|
||||
}
|
||||
|
||||
func (s *scriptedSubagentTaskRecovery) RecoverInterrupted(ctx context.Context) (int64, error) {
|
||||
if !store.IsMasterScope(ctx) || store.TenantIDFromContext(ctx) != store.MasterTenantID {
|
||||
return 0, errors.New("recovery did not receive explicit master scope")
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.calls++
|
||||
if s.calls <= s.failures {
|
||||
return 0, errors.New("database temporarily unavailable")
|
||||
}
|
||||
return s.recovered, nil
|
||||
}
|
||||
|
||||
func (s *scriptedSubagentTaskRecovery) callCount() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.calls
|
||||
}
|
||||
|
||||
func TestRecoverInterruptedSubagentTasksRetriesBeforeTraffic(t *testing.T) {
|
||||
recovery := &scriptedSubagentTaskRecovery{failures: 2, recovered: 3}
|
||||
stores := &store.Stores{SubagentTaskRecovery: recovery}
|
||||
recovered, err := recoverInterruptedSubagentTasks(
|
||||
context.Background(), stores, time.Millisecond,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("recoverInterruptedSubagentTasks: %v", err)
|
||||
}
|
||||
if recovered != 3 || recovery.callCount() != 3 {
|
||||
t.Fatalf("recovery = (%d rows, %d calls), want (3, 3)", recovered, recovery.callCount())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoverInterruptedSubagentTasksStopsOnShutdown(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
recovery := &scriptedSubagentTaskRecovery{failures: 1}
|
||||
stores := &store.Stores{SubagentTaskRecovery: recovery}
|
||||
_, err := recoverInterruptedSubagentTasks(ctx, stores, time.Hour)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("recovery error = %v, want context.Canceled", err)
|
||||
}
|
||||
if recovery.callCount() != 1 {
|
||||
t.Fatalf("recovery calls = %d, want 1", recovery.callCount())
|
||||
}
|
||||
}
|
||||
@@ -115,7 +115,7 @@ flowchart TD
|
||||
| `internal/skills/` | SKILL.md loader (5-tier hierarchy) + BM25 search + hot-reload via fsnotify |
|
||||
| `internal/channels/` | Channel manager + adapters: Telegram (forum topics, STT, bot commands), Feishu/Lark (streaming cards, media), Zalo OA, Zalo Personal, Discord, WhatsApp, Slack |
|
||||
| `internal/mcp/` | MCP server bridge (stdio, SSE, streamable-HTTP transports) |
|
||||
| `internal/scheduler/` | Lane-based concurrency control (main, subagent, cron, team lanes) with per-session serialization. Per-edition rate limits (`MaxSubagentConcurrent`, `MaxSubagentDepth`) with tenant-scoped concurrency |
|
||||
| `internal/scheduler/` | Lane-based concurrency control (main, subagent, cron, team lanes) with per-session serialization. Self-spawn and Agent Link callbacks additionally use process-wide child-run admission with per-root limits |
|
||||
| `internal/memory/` | Memory system (pgvector hybrid search) |
|
||||
| `internal/subagent/` | Subagent lifecycle: spawn, roster, task persistence (subagent_tasks table), announce queue (producer-consumer), auto-retry, per-edition rate limiting |
|
||||
| `internal/permissions/` | RBAC policy engine (admin, operator, viewer roles) |
|
||||
@@ -192,7 +192,7 @@ flowchart TD
|
||||
- **SSRF hardening (HTTPHandler)**: Caller supplies net.Dialer pinning resolved IP, blocking loopback/link-local/private ranges; no HTTP redirects (CheckRedirect returns ErrUseLastResponse)
|
||||
- **Auth header encryption**: `Authorization` + other sensitive fields in cfg.Config["headers"] encrypted at rest via AES-256-GCM; decrypted only at HTTP send-time
|
||||
- **Audit logging**: All hook invocations logged to `hook_executions` table (encrypted, PII-redacted) with dedup_key for idempotency
|
||||
- **Loop-depth guard (M5)**: SubagentStart checks recursion depth; max 3 levels prevents infinite delegation chains
|
||||
- **Hook recursion guard (M5)**: Hook dispatch tracks recursive hook re-entry and rejects it after 3 levels. Agent Link delegation uses separate admission, concurrency, and timeout safeguards.
|
||||
- **Circuit breaker**: Auto-disables hook after 3 consecutive failures in recent window (C4 mitigation)
|
||||
|
||||
### Pipeline Integration
|
||||
|
||||
@@ -111,7 +111,7 @@ Agents support three orchestration modes that determine which inter-agent tools
|
||||
- **Use case**: Agent with linked delegate targets
|
||||
- **Tools available**: `spawn`, `delegate` (dispatch to linked agents)
|
||||
- **Tools hidden**: `team_tasks`
|
||||
- **Resolution**: When `agent_links` table has rows with source = this agent
|
||||
- **Resolution**: When `agent_links` grants this agent an effective outbound delegation edge
|
||||
|
||||
### ModeTeam
|
||||
- **Use case**: Agent in a team (multiple agents collaborating)
|
||||
@@ -121,7 +121,9 @@ Agents support three orchestration modes that determine which inter-agent tools
|
||||
|
||||
**Mode Resolution Priority**: Team > Delegate > Spawn
|
||||
|
||||
The system prompt includes relevant details for each mode (delegate targets, team context, shared workspace paths).
|
||||
The system prompt includes relevant details for each mode. Agent Team runs use
|
||||
their existing shared workspace. Agent Link runs receive only delegation-scoped
|
||||
`inputs/` and `outputs/` aliases.
|
||||
|
||||
---
|
||||
|
||||
|
||||
+54
-3
@@ -144,7 +144,7 @@ Memory layers: L1 (`memory_search`) returns ranked abstracts; L2 (`memory_expand
|
||||
|
||||
| Tool | Description |
|
||||
|---|---|
|
||||
| `delegate` | Inter-agent task delegation via agent_links (async/sync modes with timeout) |
|
||||
| `delegate` | Inter-agent task delegation via `agent_links` (async/sync, optional structured input files, isolated output publication) |
|
||||
|
||||
### Teams (`group:team`)
|
||||
|
||||
@@ -170,6 +170,19 @@ Memory layers: L1 (`memory_search`) returns ranked abstracts; L2 (`memory_expand
|
||||
| `read_document` | Extract and analyze documents (PDF, DOCX, images). Supports local-first extraction via pdftotext/pandoc before falling back to cloud vision (opt-in via config) |
|
||||
| `read_video` | Analyze/transcribe video content |
|
||||
|
||||
Image attachments keep absolute paths only in internal `MediaRef` storage.
|
||||
Model-visible `<media:image>` tags expose an exact media ID plus a logical path
|
||||
such as `.uploads/photo.jpg` or delegated `inputs/photo.jpg`. Image tools resolve
|
||||
those values against the active workspace; they never infer another agent's
|
||||
absolute workspace path. Unknown image IDs fail instead of falling back to an
|
||||
unrelated image. Exact `media_id` lookup is an in-process convenience; Claude
|
||||
CLI/MCP uses the workspace-confined logical `path` under signed bridge context.
|
||||
|
||||
For Agent Link artifact runs, Claude CLI MCP requests carry a signed delegation
|
||||
ID and staged-input root. Bridge tools therefore apply the same read-only
|
||||
`inputs/` policy as in-process tools, and generated media remains inside
|
||||
`outputs/` until the delegation manifest is validated and atomically published.
|
||||
|
||||
### Skills & Content
|
||||
|
||||
| Tool | Description |
|
||||
@@ -613,13 +626,51 @@ stateDiagram-v2
|
||||
|
||||
| Constraint | Default |
|
||||
|---|---|
|
||||
| Max concurrent | 8 (across all parents) |
|
||||
| Max concurrent | 20 executing descendants per root agent |
|
||||
| Max spawn depth | 1 |
|
||||
| Max children per agent | 5 |
|
||||
| Archive after | 60 min |
|
||||
| Max iterations | 20 per subagent |
|
||||
|
||||
**Actions:** `spawn` (async, returns immediately), `run` (sync, blocks until done), `list`, `cancel` (by ID / `"all"` / `"last"`), `steer` (cancel + respawn with new message).
|
||||
`maxSpawnDepth` applies only to an agent's own subagent tree. Agent Link
|
||||
delegation starts a new tree whose spawn depth, concurrency, fanout, retry, and
|
||||
model settings are resolved from the target agent.
|
||||
|
||||
**Actions:** `spawn(action="spawn", mode="async"|"sync")`, `get`, `list`,
|
||||
`cancel` (by ID / `"all"` / `"last"`), `steer` (cancel + respawn with new
|
||||
message), and `wait`.
|
||||
|
||||
An accepted async spawn returns its short runtime `task_id` plus a durable
|
||||
`completion_id`. Terminal status, the full text result, and workspace-safe
|
||||
logical media descriptors are written to `subagent_tasks` before the parent
|
||||
announcement is attempted. Announcement delivery retries a bounded number of
|
||||
times; if the inbound queue is still full or the gateway restarts after
|
||||
terminal persistence, the owning root agent can recover the result and logical
|
||||
file paths with:
|
||||
|
||||
```text
|
||||
spawn(action="get", completion_id="<uuid>")
|
||||
```
|
||||
|
||||
The lookup is scoped by tenant and immutable root-agent UUID. A same-key agent
|
||||
created later, another agent in the tenant, and delegation completion rows
|
||||
cannot satisfy the lookup.
|
||||
|
||||
Terminal persistence uses per-attempt database deadlines and a longer bounded
|
||||
retry window than announcement delivery. If the database remains unavailable
|
||||
for the entire window, GoClaw does not falsely mark the announcement as
|
||||
delivered; a live announcement is still attempted. On the next startup that
|
||||
can reach the database, every non-terminal completion row left by the previous
|
||||
process is marked `failed` with an interruption reason, so it cannot remain
|
||||
`queued` or `running` forever. The single-process gateway retries this
|
||||
reconciliation before accepting traffic when the database is temporarily
|
||||
unavailable. Graceful gateway shutdown drains this completion lifecycle before
|
||||
provider and database teardown.
|
||||
|
||||
Self-spawn and Agent Link callbacks also share a process safety cap (Standard:
|
||||
32; Lite: 2) and a bounded pending queue of 128. Inside an Agent Link artifact
|
||||
run, async `spawn` and async nested `delegate` are rejected; configured
|
||||
synchronous descendants complete before the outer artifact is published.
|
||||
|
||||
Subagents share the same `SecureCLIStore` as their parent — the credentialed binary gate cannot be bypassed by delegating exec to a child.
|
||||
|
||||
|
||||
+9
-3
@@ -461,13 +461,19 @@ Browser pairing allows web UI clients to authenticate without full admin credent
|
||||
|
||||
## 12. Delegation Security
|
||||
|
||||
Agent delegation is protected through delegation history tracking and concurrency controls.
|
||||
Agent delegation is protected through isolated artifact exchange, directional
|
||||
link permission checks, and bounded child-run admission.
|
||||
|
||||
| Control | Scope | Description |
|
||||
|---------|-------|-------------|
|
||||
| Per-agent load cap | B (all sources) | `other_config.max_delegation_load` limits total concurrent delegations targeting B |
|
||||
| Artifact boundary | One Agent Link run | Staged inputs are read-only; the delegatee writes only to ephemeral outputs that are validated before atomic publish |
|
||||
| Per-root limit | One root agent's self-spawn tree | Agent `subagents.maxConcurrent`, default 20 |
|
||||
| Process safety cap | All self-spawn and Agent Link callbacks | Standard 32 / Lite 2, with at most 128 independent pending chains |
|
||||
| Nested lifetime | Agent Link artifact run | Async descendants are rejected; sync descendants complete before publication |
|
||||
| Durable async result | Source/root agent UUID within one tenant | Terminal text and logical media paths use deadline-bounded persistence retries before announcement; single-process startup retries pre-traffic recovery that marks previous-process non-terminal rows failed, and successful terminal writes remain explicitly retrievable by completion UUID |
|
||||
|
||||
When concurrency limits are hit, the error message is written for LLM reasoning: *"Agent at capacity (5/5). Try a different agent or handle it yourself."*
|
||||
`agent_links.max_concurrent` is retained as compatibility metadata and is not
|
||||
currently enforced by runtime admission.
|
||||
|
||||
---
|
||||
|
||||
|
||||
+52
-6
@@ -579,10 +579,17 @@ Each delegation link (lead→member) has its own settings:
|
||||
|
||||
| Layer | Scope | Default |
|
||||
|-------|-------|---------|
|
||||
| Per-link | Simultaneous delegations from lead to a specific member | 3 |
|
||||
| Per-agent | Total concurrent delegations targeting any single member | 5 |
|
||||
| Per-root agent | Executing self-spawn descendants across the full tree | 20 |
|
||||
| Process safety cap | Executing self-spawn and Agent Link callbacks | Standard 32 / Lite 2 |
|
||||
| Pending admission | Independent child-run chains waiting for capacity | 128 |
|
||||
|
||||
When limits are hit, the error message is written for LLM reasoning: "Agent at capacity (5/5). Try a different agent or handle it yourself."
|
||||
Each Agent Link delegation begins a new target-owned spawn tree. The source
|
||||
agent's `maxSpawnDepth` does not limit the target agent; any subagents created by
|
||||
the target use the target's own subagent settings.
|
||||
|
||||
`agent_links.max_concurrent` remains serialized compatibility metadata and is
|
||||
not currently an admission limit. Agent Team task scheduling keeps its existing
|
||||
lane/workspace contract.
|
||||
|
||||
---
|
||||
|
||||
@@ -593,7 +600,7 @@ Agent links define directed delegation relationships between agents, separate fr
|
||||
- **Target agent** (can receive delegations from)
|
||||
- **Direction** (outbound from source, inbound to target)
|
||||
- **Team context** (optional `team_id` if created by team setup)
|
||||
- **Concurrency limit** (max simultaneous delegations for this link)
|
||||
- **Reserved concurrency metadata** (`max_concurrent`; retained for compatibility, not currently enforced)
|
||||
|
||||
**Team-created links** are automatically created when a team is set up (lead → each member). Links remain even if the team is deleted or members are removed, allowing manual cleanup.
|
||||
|
||||
@@ -601,9 +608,48 @@ Agent links define directed delegation relationships between agents, separate fr
|
||||
|
||||
## 12. Delegation Context
|
||||
|
||||
### SenderID Clearing
|
||||
### Authorization and artifact scope
|
||||
|
||||
In sync delegations, the delegate agent's context has the `senderID` cleared. This is critical because delegations are system-initiated — the delegate should not inherit the caller's group writer permissions, which would incorrectly deny file writes. Each delegate agent has its own writer list.
|
||||
Agent Link dispatch preserves the caller's authorization scope for policy
|
||||
checks but attenuates filesystem authority. The delegatee receives staged
|
||||
read-only copies through `inputs/` and writes only to `outputs/`. After the run,
|
||||
regular-file, path, symlink, count, and size validation completes before an
|
||||
atomic publication to the caller's `.delegations/<delegation-id>/` directory.
|
||||
Agent Team shared-workspace behavior is unchanged.
|
||||
|
||||
Successful publications persist with `manifest.json` and logical `outputs/...`
|
||||
entries. Failed/cancelled exchanges are retained for 60 minutes for diagnostics
|
||||
and removed by a restart-safe janitor whose per-sweep item and filesystem-entry
|
||||
work are both bounded. Traces correlate by
|
||||
`delegation_id` and record only logical paths, sizes, hashes, status, and
|
||||
timestamps—never physical exchange or caller host paths.
|
||||
|
||||
Async Agent Link dispatch also creates a durable completion row before the
|
||||
child-run ticket is activated. The full terminal result and logical published
|
||||
artifact paths are persisted before the automatic parent announcement. If
|
||||
bounded announcement retries cannot enter the inbound queue, the source agent
|
||||
can retrieve the result after the run—or after a gateway restart—with:
|
||||
|
||||
```text
|
||||
delegate(action="get", delegation_id="<uuid>")
|
||||
```
|
||||
|
||||
Retrieval is scoped by tenant and the immutable UUID of the source agent. It
|
||||
does not grant the target agent, another linked source, or a later same-key
|
||||
agent access to the result. Delegation completion rows are excluded from
|
||||
`/subagents` and self-clone roster queries.
|
||||
|
||||
Terminal database writes use per-attempt deadlines plus an extended bounded
|
||||
retry window. If the database remains unavailable for that whole window, the
|
||||
live announcement is still attempted but is not recorded as durably delivered;
|
||||
a crash during the same outage may prevent the terminal result from being
|
||||
recorded. On the next startup that can reach the database, every non-terminal
|
||||
completion row from the previous process is marked `failed` with an interruption
|
||||
reason instead of remaining `queued` or `running` forever. The gateway retries
|
||||
that reconciliation before accepting traffic if the database is temporarily
|
||||
unavailable. The current runtime assumes one active gateway process. Graceful
|
||||
shutdown waits for accepted completion persistence before tearing down runtime
|
||||
dependencies.
|
||||
|
||||
### Trace Linking
|
||||
|
||||
|
||||
@@ -166,13 +166,20 @@ Uploaded files (from web chat, Telegram, Discord, etc.) are persisted to:
|
||||
|
||||
Uploads without a usable original filename fall back to `{uuid}.{ext}`. Legacy media refs may still resolve from `.media/{sessionHash}/{uuid}.{ext}`.
|
||||
|
||||
The `enrichDocumentPaths()` function injects the full path into `<media:document>` tags:
|
||||
The `enrichDocumentPaths()` function injects the exact media ID and a logical
|
||||
path relative to the active agent workspace into `<media:document>` tags:
|
||||
|
||||
```
|
||||
<media:document name="report.pdf" path="/app/workspace/.uploads/report-a1b2c3d4.pdf">
|
||||
<media:document name="report.pdf" id="..." path=".uploads/report-a1b2c3d4.pdf">
|
||||
```
|
||||
|
||||
Agents can read these files directly via exec — no copy to `/tmp` needed. For archive uploads such as `.zip`, inspect or extract with commands like `unzip -l "<path>"` or `unzip -q "<path>" -d <output-dir>`.
|
||||
Normal agent runs can read these workspace files directly via `exec` — no copy
|
||||
to `/tmp` is needed. Agent Link delegations instead receive selected files as
|
||||
read-only `inputs/...` paths in an isolated delegation exchange. Their `exec`
|
||||
calls fail closed unless an active sandbox is available, and generated files
|
||||
must be written under `outputs/` for validation and publication back to the
|
||||
caller. For archive uploads such as `.zip`, inspect or extract them within the
|
||||
authorized workspace or delegation paths.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/providers"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/tracing"
|
||||
)
|
||||
|
||||
func isExistingRealDirectory(path string) bool {
|
||||
if path == "" {
|
||||
return false
|
||||
}
|
||||
info, err := os.Lstat(path)
|
||||
return err == nil && info.IsDir() && info.Mode()&os.ModeSymlink == 0
|
||||
}
|
||||
|
||||
func validateDelegationArtifactWorkspace(delegationID, inputsPath, outputsPath string) bool {
|
||||
parsedID, err := uuid.Parse(delegationID)
|
||||
if err != nil || parsedID == uuid.Nil ||
|
||||
!isExistingRealDirectory(inputsPath) ||
|
||||
!isExistingRealDirectory(outputsPath) {
|
||||
return false
|
||||
}
|
||||
inputsReal, err := filepath.EvalSymlinks(filepath.Clean(inputsPath))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
outputsReal, err := filepath.EvalSymlinks(filepath.Clean(outputsPath))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
exchangeRoot := filepath.Dir(inputsReal)
|
||||
return filepath.Base(inputsReal) == "inputs" &&
|
||||
filepath.Base(outputsReal) == "outputs" &&
|
||||
filepath.Dir(outputsReal) == exchangeRoot &&
|
||||
filepath.Base(exchangeRoot) == parsedID.String() &&
|
||||
filepath.Base(filepath.Dir(exchangeRoot)) == "delegations" &&
|
||||
filepath.Base(filepath.Dir(filepath.Dir(exchangeRoot))) == "collaboration"
|
||||
}
|
||||
|
||||
func delegationArtifactTextRedactor(req *RunRequest) tracing.TextRedactor {
|
||||
if req == nil || req.RunKind != "delegate" ||
|
||||
req.DelegateInputsPath == "" || req.DelegateOutputsPath == "" {
|
||||
return nil
|
||||
}
|
||||
type replacement struct {
|
||||
value string
|
||||
alias string
|
||||
}
|
||||
var replacements []replacement
|
||||
seen := make(map[string]struct{})
|
||||
add := func(value, alias string) {
|
||||
for _, variant := range []string{
|
||||
value,
|
||||
filepath.ToSlash(value),
|
||||
strings.Trim(strconv.Quote(value), `"`),
|
||||
} {
|
||||
if variant == "" {
|
||||
continue
|
||||
}
|
||||
if _, exists := seen[variant]; exists {
|
||||
continue
|
||||
}
|
||||
seen[variant] = struct{}{}
|
||||
replacements = append(replacements, replacement{value: variant, alias: alias})
|
||||
}
|
||||
}
|
||||
add(req.DelegateInputsPath, "inputs")
|
||||
add(req.DelegateOutputsPath, "outputs")
|
||||
if root := filepath.Dir(req.DelegateInputsPath); root == filepath.Dir(req.DelegateOutputsPath) {
|
||||
add(root, "delegation exchange")
|
||||
}
|
||||
sort.Slice(replacements, func(i, j int) bool {
|
||||
return len(replacements[i].value) > len(replacements[j].value)
|
||||
})
|
||||
pairs := make([]string, 0, len(replacements)*2)
|
||||
for _, replacement := range replacements {
|
||||
pairs = append(pairs, replacement.value, replacement.alias)
|
||||
}
|
||||
replacer := strings.NewReplacer(pairs...)
|
||||
return replacer.Replace
|
||||
}
|
||||
|
||||
func withDelegationArtifactTextRedactor(ctx context.Context, req *RunRequest) context.Context {
|
||||
return tracing.WithTextRedactor(ctx, delegationArtifactTextRedactor(req))
|
||||
}
|
||||
|
||||
func redactDelegationAgentEvent(req *RunRequest, event AgentEvent) AgentEvent {
|
||||
redactor := delegationArtifactTextRedactor(req)
|
||||
event.Payload = tracing.RedactValueWith(redactor, event.Payload)
|
||||
return event
|
||||
}
|
||||
|
||||
func redactDelegationMessage(req *RunRequest, message providers.Message) providers.Message {
|
||||
redactor := delegationArtifactTextRedactor(req)
|
||||
if redactor == nil {
|
||||
return message
|
||||
}
|
||||
message.Content = redactor(message.Content)
|
||||
message.Thinking = redactor(message.Thinking)
|
||||
if len(message.RawAssistantContent) > 0 {
|
||||
message.RawAssistantContent = json.RawMessage(redactor(string(message.RawAssistantContent)))
|
||||
}
|
||||
for i := range message.ToolCalls {
|
||||
if arguments, ok := tracing.RedactValueWith(redactor, message.ToolCalls[i].Arguments).(map[string]any); ok {
|
||||
message.ToolCalls[i].Arguments = arguments
|
||||
}
|
||||
message.ToolCalls[i].ParseError = redactor(message.ToolCalls[i].ParseError)
|
||||
}
|
||||
// Pre-publication paths are ephemeral and cannot be durable session media.
|
||||
message.MediaRefs = nil
|
||||
return message
|
||||
}
|
||||
|
||||
func redactDelegationRunResult(req *RunRequest, result *RunResult) *RunResult {
|
||||
if result == nil {
|
||||
return nil
|
||||
}
|
||||
redactor := delegationArtifactTextRedactor(req)
|
||||
if redactor == nil {
|
||||
return result
|
||||
}
|
||||
result.Content = redactor(result.Content)
|
||||
result.Thinking = redactor(result.Thinking)
|
||||
result.LastBlockReply = redactor(result.LastBlockReply)
|
||||
for i := range result.Deliverables {
|
||||
result.Deliverables[i] = redactor(result.Deliverables[i])
|
||||
}
|
||||
// The outer artifact publisher is the only egress path.
|
||||
result.Media = nil
|
||||
return result
|
||||
}
|
||||
@@ -27,6 +27,32 @@ type contextSetupResult struct {
|
||||
// values needed by the agent loop and tool execution. Also runs input guard and message
|
||||
// truncation. Returns error only if input guard blocks the message.
|
||||
func (l *Loop) injectContext(ctx context.Context, req *RunRequest) (contextSetupResult, error) {
|
||||
isArtifactDelegation := req.RunKind == "delegate"
|
||||
|
||||
// A nested run must not inherit filesystem, Team, media, or delegation
|
||||
// authority from its caller. Install explicit empty values before resolving
|
||||
// this run's own scope.
|
||||
ctx = store.WithRunContext(ctx, nil)
|
||||
ctx = tools.WithToolWorkspace(ctx, "")
|
||||
ctx = tools.WithToolTeamWorkspace(ctx, "")
|
||||
ctx = tools.WithToolTeamRoot(ctx, "")
|
||||
ctx = tools.WithToolTeamID(ctx, "")
|
||||
ctx = tools.WithTeamTaskID(ctx, "")
|
||||
ctx = tools.WithLeaderAgentID(ctx, "")
|
||||
ctx = tools.WithTenantAllowedPaths(ctx, nil)
|
||||
ctx = tools.WithWorkspaceChannel(ctx, "")
|
||||
ctx = tools.WithWorkspaceChatID(ctx, "")
|
||||
ctx = tools.WithDelegationID(ctx, "")
|
||||
ctx = tools.WithDelegationArtifactInputs(ctx, "")
|
||||
ctx = tools.WithRunKind(ctx, "")
|
||||
ctx = tools.WithRunMediaPaths(ctx, nil)
|
||||
ctx = tools.WithRunMediaNames(ctx, nil)
|
||||
ctx = tools.WithMediaImages(ctx, nil)
|
||||
ctx = tools.WithMediaImageRefs(ctx, nil)
|
||||
ctx = tools.WithMediaDocRefs(ctx, nil)
|
||||
ctx = tools.WithMediaAudioRefs(ctx, nil)
|
||||
ctx = tools.WithMediaVideoRefs(ctx, nil)
|
||||
|
||||
// Inject agent UUID + key into context for tool routing
|
||||
if l.agentUUID != uuid.Nil {
|
||||
ctx = store.WithAgentID(ctx, l.agentUUID)
|
||||
@@ -87,7 +113,7 @@ func (l *Loop) injectContext(ctx context.Context, req *RunRequest) (contextSetup
|
||||
ctx = tools.WithTenantToolSettings(ctx, l.tenantToolSettings)
|
||||
}
|
||||
// Inject tenant-specific allowed paths for filesystem tools.
|
||||
if len(l.tenantAllowedPaths) > 0 {
|
||||
if !isArtifactDelegation && len(l.tenantAllowedPaths) > 0 {
|
||||
ctx = tools.WithTenantAllowedPaths(ctx, l.tenantAllowedPaths)
|
||||
}
|
||||
// Inject channel type into context for tools (e.g. message tool needs it for Zalo group routing)
|
||||
@@ -150,6 +176,9 @@ func (l *Loop) injectContext(ctx context.Context, req *RunRequest) (contextSetup
|
||||
if req.DelegationID != "" {
|
||||
ctx = tools.WithDelegationID(ctx, req.DelegationID)
|
||||
}
|
||||
if req.RunKind != "" {
|
||||
ctx = tools.WithRunKind(ctx, req.RunKind)
|
||||
}
|
||||
|
||||
// --- Per-user setup: file seeding + workspace resolution ---
|
||||
// Uses userSetups sync.Map to track both concerns atomically per user.
|
||||
@@ -163,7 +192,7 @@ func (l *Loop) injectContext(ctx context.Context, req *RunRequest) (contextSetup
|
||||
// Layer order: tenant → team → project (future) → user/chat
|
||||
// Two entry modes: solo agent (base = l.workspace) or team context (base = l.dataDir).
|
||||
// Result is always a single folder set via WithToolWorkspace.
|
||||
if l.workspace != "" && req.UserID != "" {
|
||||
if !isArtifactDelegation && l.workspace != "" && req.UserID != "" {
|
||||
ws := setup.workspace
|
||||
if ws == "" {
|
||||
ws = l.workspace
|
||||
@@ -197,12 +226,21 @@ func (l *Loop) injectContext(ctx context.Context, req *RunRequest) (contextSetup
|
||||
effectiveWorkspace = l.workspace
|
||||
}
|
||||
ctx = tools.WithToolWorkspace(ctx, effectiveWorkspace)
|
||||
} else if l.workspace != "" {
|
||||
} else if !isArtifactDelegation && l.workspace != "" {
|
||||
ctx = tools.WithToolWorkspace(ctx, l.workspace)
|
||||
}
|
||||
|
||||
if isArtifactDelegation {
|
||||
if req.TeamWorkspace != "" ||
|
||||
!validateDelegationArtifactWorkspace(req.DelegationID, req.DelegateInputsPath, req.DelegateOutputsPath) {
|
||||
return contextSetupResult{}, fmt.Errorf("invalid delegation artifact workspace")
|
||||
}
|
||||
ctx = tools.WithDelegationArtifactInputs(ctx, req.DelegateInputsPath)
|
||||
ctx = tools.WithToolWorkspace(ctx, req.DelegateOutputsPath)
|
||||
}
|
||||
|
||||
// Team workspace: dispatched task overrides default workspace.
|
||||
if req.TeamWorkspace != "" {
|
||||
if !isArtifactDelegation && req.TeamWorkspace != "" {
|
||||
if err := os.MkdirAll(req.TeamWorkspace, 0755); err != nil {
|
||||
// See note above on loop_context user workspace fallback. A broken
|
||||
// req.TeamWorkspace would otherwise become cmd.Dir and surface as
|
||||
@@ -214,7 +252,7 @@ func (l *Loop) injectContext(ctx context.Context, req *RunRequest) (contextSetup
|
||||
ctx = tools.WithToolWorkspace(ctx, req.TeamWorkspace)
|
||||
}
|
||||
}
|
||||
if req.TeamID != "" {
|
||||
if !isArtifactDelegation && req.TeamID != "" {
|
||||
ctx = tools.WithToolTeamID(ctx, req.TeamID)
|
||||
// Team root for dispatched tasks: resolve the UserChatLayer-stripped root
|
||||
// so the dispatched agent can still read peer-scoped files in the same team.
|
||||
@@ -228,7 +266,7 @@ func (l *Loop) injectContext(ctx context.Context, req *RunRequest) (contextSetup
|
||||
ctx = tools.WithToolTeamRoot(ctx, teamRoot)
|
||||
}
|
||||
}
|
||||
if req.LeaderAgentID != "" {
|
||||
if !isArtifactDelegation && req.LeaderAgentID != "" {
|
||||
ctx = tools.WithLeaderAgentID(ctx, req.LeaderAgentID)
|
||||
}
|
||||
|
||||
@@ -237,14 +275,14 @@ func (l *Loop) injectContext(ctx context.Context, req *RunRequest) (contextSetup
|
||||
var resolvedTeamSettings json.RawMessage
|
||||
// Dispatched tasks already have TeamWorkspace set but still need team settings
|
||||
// for TeamIsolated flag. Fetch by explicit TeamID in that branch.
|
||||
if req.TeamWorkspace != "" && req.TeamID != "" && l.teamStore != nil {
|
||||
if !isArtifactDelegation && req.TeamWorkspace != "" && req.TeamID != "" && l.teamStore != nil {
|
||||
if teamUUID, err := uuid.Parse(req.TeamID); err == nil {
|
||||
if team, _ := l.teamStore.GetTeam(ctx, teamUUID); team != nil {
|
||||
resolvedTeamSettings = team.Settings
|
||||
}
|
||||
}
|
||||
}
|
||||
if req.TeamWorkspace == "" && l.teamStore != nil && l.agentUUID != uuid.Nil {
|
||||
if !isArtifactDelegation && req.TeamWorkspace == "" && l.teamStore != nil && l.agentUUID != uuid.Nil {
|
||||
if team, _ := l.teamStore.GetTeamForAgent(ctx, l.agentUUID); team != nil {
|
||||
resolvedTeamSettings = team.Settings
|
||||
wsChat := req.ChatID
|
||||
@@ -285,7 +323,16 @@ func (l *Loop) injectContext(ctx context.Context, req *RunRequest) (contextSetup
|
||||
}
|
||||
|
||||
// V3 workspace: resolve once, set immutable context.
|
||||
{
|
||||
if isArtifactDelegation {
|
||||
ctx = workspace.WithContext(ctx, &workspace.WorkspaceContext{
|
||||
ActivePath: req.DelegateOutputsPath,
|
||||
Scope: workspace.ScopeDelegate,
|
||||
MemoryScope: "user",
|
||||
KGScope: "user",
|
||||
OwnerID: req.UserID,
|
||||
EnforcementLabel: workspace.DefaultEnforcementLabel(workspace.ScopeDelegate, false),
|
||||
})
|
||||
} else {
|
||||
var teamIDPtr *string
|
||||
if req.TeamID != "" {
|
||||
teamIDPtr = &req.TeamID
|
||||
@@ -386,6 +433,10 @@ func (l *Loop) injectContext(ctx context.Context, req *RunRequest) (contextSetup
|
||||
}
|
||||
// Extract resolved credential user ID (set earlier via WithCredentialUserID, empty if not resolved).
|
||||
credUserID := store.ExplicitCredentialUserIDFromContext(ctx)
|
||||
tenantAllowedPaths := l.tenantAllowedPaths
|
||||
if isArtifactDelegation {
|
||||
tenantAllowedPaths = nil
|
||||
}
|
||||
rc := &store.RunContext{
|
||||
AgentID: l.agentUUID,
|
||||
AgentKey: l.id,
|
||||
@@ -419,9 +470,10 @@ func (l *Loop) injectContext(ctx context.Context, req *RunRequest) (contextSetup
|
||||
WorkspaceChatID: effectiveWorkspaceChatID,
|
||||
TeamIsolated: resolvedTeamSettings != nil && !tools.IsSharedWorkspace(resolvedTeamSettings),
|
||||
TeamTaskID: req.TeamTaskID,
|
||||
DelegationID: req.DelegationID,
|
||||
LeaderAgentID: tools.LeaderAgentIDFromCtx(ctx),
|
||||
AgentToolKey: l.id,
|
||||
TenantAllowedPaths: l.tenantAllowedPaths,
|
||||
TenantAllowedPaths: tenantAllowedPaths,
|
||||
}
|
||||
ctx = store.WithRunContext(ctx, rc)
|
||||
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/providers"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/tools"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/workspace"
|
||||
)
|
||||
|
||||
func newArtifactRunRequest(t *testing.T, root string) *RunRequest {
|
||||
t.Helper()
|
||||
delegationID := uuid.NewString()
|
||||
exchange := filepath.Join(root, "collaboration", "delegations", delegationID)
|
||||
inputs := filepath.Join(exchange, "inputs")
|
||||
outputs := filepath.Join(exchange, "outputs")
|
||||
if err := os.MkdirAll(inputs, 0750); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.MkdirAll(outputs, 0750); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &RunRequest{
|
||||
SessionKey: "agent:worker:delegate:test",
|
||||
RunKind: "delegate",
|
||||
DelegationID: delegationID,
|
||||
DelegateInputsPath: inputs,
|
||||
DelegateOutputsPath: outputs,
|
||||
}
|
||||
}
|
||||
|
||||
func newArtifactTestLoop(root string) *Loop {
|
||||
return NewLoop(LoopConfig{
|
||||
ID: "worker",
|
||||
AgentUUID: uuid.New(),
|
||||
TenantID: store.MasterTenantID,
|
||||
Workspace: filepath.Join(root, "agents", "worker"),
|
||||
DataDir: root,
|
||||
Sessions: &nopSessionStore{},
|
||||
})
|
||||
}
|
||||
|
||||
func TestInjectContext_UsesOnlyDelegationArtifactWorkspace(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
req := newArtifactRunRequest(t, root)
|
||||
parentCtx := tools.WithToolTeamWorkspace(context.Background(), filepath.Join(root, "teams", "stale"))
|
||||
parentCtx = tools.WithToolTeamID(parentCtx, uuid.NewString())
|
||||
parentCtx = tools.WithTenantAllowedPaths(parentCtx, []string{filepath.Join(root, "tenant-allowed")})
|
||||
parentCtx = tools.WithDelegationArtifactInputs(parentCtx, filepath.Join(root, "stale-inputs"))
|
||||
parentCtx = tools.WithRunMediaPaths(parentCtx, []string{"/stale/upload.txt"})
|
||||
|
||||
setup, err := newArtifactTestLoop(root).injectContext(parentCtx, req)
|
||||
if err != nil {
|
||||
t.Fatalf("injectContext: %v", err)
|
||||
}
|
||||
if got := tools.ToolWorkspaceFromCtx(setup.ctx); got != req.DelegateOutputsPath {
|
||||
t.Fatalf("workspace = %q, want outputs %q", got, req.DelegateOutputsPath)
|
||||
}
|
||||
if got := tools.DelegationArtifactInputsFromCtx(setup.ctx); got != req.DelegateInputsPath {
|
||||
t.Fatalf("inputs = %q, want %q", got, req.DelegateInputsPath)
|
||||
}
|
||||
if got := tools.ToolTeamWorkspaceFromCtx(setup.ctx); got != "" {
|
||||
t.Fatalf("parent Team workspace survived: %q", got)
|
||||
}
|
||||
if got := tools.ToolTeamIDFromCtx(setup.ctx); got != "" {
|
||||
t.Fatalf("parent Team ID survived: %q", got)
|
||||
}
|
||||
if got := tools.TenantAllowedPathsFromCtx(setup.ctx); len(got) != 0 {
|
||||
t.Fatalf("parent tenant allowed paths survived: %v", got)
|
||||
}
|
||||
if got := tools.RunMediaPathsFromCtx(setup.ctx); len(got) != 0 {
|
||||
t.Fatalf("parent media paths survived: %v", got)
|
||||
}
|
||||
wc := workspace.FromContext(setup.ctx)
|
||||
if wc == nil || wc.Scope != workspace.ScopeDelegate || wc.ActivePath != req.DelegateOutputsPath {
|
||||
t.Fatalf("workspace context = %#v", wc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInjectContext_DelegationSetupFailsClosed(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
req := newArtifactRunRequest(t, root)
|
||||
req.DelegateOutputsPath = filepath.Join(root, "missing")
|
||||
if _, err := newArtifactTestLoop(root).injectContext(context.Background(), req); err == nil {
|
||||
t.Fatal("missing artifact workspace fell back to personal workspace")
|
||||
}
|
||||
|
||||
outside := filepath.Join(root, "outside")
|
||||
if err := os.MkdirAll(outside, 0750); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
link := filepath.Join(root, "output-link")
|
||||
if err := os.Symlink(outside, link); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.DelegateOutputsPath = link
|
||||
if _, err := newArtifactTestLoop(root).injectContext(context.Background(), req); err == nil {
|
||||
t.Fatal("symlink artifact workspace fell back to personal workspace")
|
||||
}
|
||||
|
||||
req = newArtifactRunRequest(t, root)
|
||||
req.DelegationID = uuid.NewString()
|
||||
if _, err := newArtifactTestLoop(root).injectContext(context.Background(), req); err == nil {
|
||||
t.Fatal("exchange path not bound to delegation ID was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInjectContext_DelegatePreservesAuthorizationScope(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
req := newArtifactRunRequest(t, root)
|
||||
req.UserID = "group:telegram:-100123"
|
||||
req.SenderID = "386246614"
|
||||
req.Role = "viewer"
|
||||
req.Channel = "delegate"
|
||||
req.ChannelType = "telegram"
|
||||
req.ChatID = "-100123"
|
||||
req.PeerKind = "group"
|
||||
req.WorkspaceChannel = "telegram-main"
|
||||
req.WorkspaceChatID = "-100123"
|
||||
|
||||
setup, err := newArtifactTestLoop(root).injectContext(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("injectContext: %v", err)
|
||||
}
|
||||
if got := store.UserIDFromContext(setup.ctx); got != req.UserID {
|
||||
t.Fatalf("UserID = %q, want %q", got, req.UserID)
|
||||
}
|
||||
if got := store.SenderIDFromContext(setup.ctx); got != req.SenderID {
|
||||
t.Fatalf("SenderID = %q, want %q", got, req.SenderID)
|
||||
}
|
||||
if got := store.RoleFromContext(setup.ctx); got != req.Role {
|
||||
t.Fatalf("Role = %q, want %q", got, req.Role)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInjectContext_TeamDispatchKeepsSharedWorkspaceContract(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
teamWorkspace := filepath.Join(root, "teams", "team-a", "shared")
|
||||
if err := os.MkdirAll(teamWorkspace, 0750); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req := &RunRequest{
|
||||
SessionKey: "agent:worker:team:test",
|
||||
TeamWorkspace: teamWorkspace,
|
||||
TeamID: uuid.NewString(),
|
||||
TeamTaskID: uuid.NewString(),
|
||||
LeaderAgentID: uuid.NewString(),
|
||||
}
|
||||
|
||||
setup, err := newArtifactTestLoop(root).injectContext(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("injectContext: %v", err)
|
||||
}
|
||||
if got := tools.ToolWorkspaceFromCtx(setup.ctx); got != teamWorkspace {
|
||||
t.Fatalf("active workspace = %q, want Team workspace %q", got, teamWorkspace)
|
||||
}
|
||||
if got := tools.ToolTeamWorkspaceFromCtx(setup.ctx); got != teamWorkspace {
|
||||
t.Fatalf("Team workspace = %q, want %q", got, teamWorkspace)
|
||||
}
|
||||
if got := tools.ToolTeamIDFromCtx(setup.ctx); got != req.TeamID {
|
||||
t.Fatalf("Team ID = %q, want %q", got, req.TeamID)
|
||||
}
|
||||
if got := tools.TeamTaskIDFromCtx(setup.ctx); got != req.TeamTaskID {
|
||||
t.Fatalf("Team task ID = %q, want %q", got, req.TeamTaskID)
|
||||
}
|
||||
if got := tools.DelegationArtifactInputsFromCtx(setup.ctx); got != "" {
|
||||
t.Fatalf("Team dispatch gained artifact inputs: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelegationArtifactRedactionCoversEventsSessionsAndResults(t *testing.T) {
|
||||
req := newArtifactRunRequest(t, t.TempDir())
|
||||
hostResult := filepath.Join(req.DelegateOutputsPath, "report.txt")
|
||||
hostInput := filepath.Join(req.DelegateInputsPath, "source.txt")
|
||||
|
||||
event := redactDelegationAgentEvent(req, AgentEvent{
|
||||
Payload: map[string]any{
|
||||
"content": "created " + hostResult,
|
||||
"media": []any{hostResult},
|
||||
},
|
||||
})
|
||||
message := redactDelegationMessage(req, providers.Message{
|
||||
Content: "read " + hostInput,
|
||||
Thinking: "write " + hostResult,
|
||||
MediaRefs: []providers.MediaRef{{Path: hostResult}},
|
||||
ToolCalls: []providers.ToolCall{{
|
||||
Arguments: map[string]any{"path": hostResult},
|
||||
}},
|
||||
RawAssistantContent: json.RawMessage(`{"path":` + mustJSONQuote(t, hostResult) + `}`),
|
||||
})
|
||||
result := redactDelegationRunResult(req, &RunResult{
|
||||
Content: "done at " + hostResult,
|
||||
Thinking: hostInput,
|
||||
Deliverables: []string{hostResult},
|
||||
Media: []MediaResult{{Path: hostResult}},
|
||||
})
|
||||
|
||||
encoded, err := json.Marshal([]any{event, message, result})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, hostRoot := range []string{
|
||||
filepath.Dir(req.DelegateInputsPath),
|
||||
req.DelegateInputsPath,
|
||||
req.DelegateOutputsPath,
|
||||
} {
|
||||
if strings.Contains(string(encoded), hostRoot) {
|
||||
t.Fatalf("redacted boundary leaked %q: %s", hostRoot, encoded)
|
||||
}
|
||||
}
|
||||
if len(message.MediaRefs) != 0 || len(result.Media) != 0 {
|
||||
t.Fatalf("ephemeral media survived: message=%#v result=%#v", message.MediaRefs, result.Media)
|
||||
}
|
||||
if !strings.Contains(string(encoded), "inputs/source.txt") ||
|
||||
!strings.Contains(string(encoded), "outputs/report.txt") {
|
||||
t.Fatalf("logical aliases missing: %s", encoded)
|
||||
}
|
||||
}
|
||||
|
||||
func mustJSONQuote(t *testing.T, value string) string {
|
||||
t.Helper()
|
||||
encoded, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(encoded)
|
||||
}
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
"github.com/nextlevelbuilder/goclaw/internal/i18n"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/providers"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/tools"
|
||||
)
|
||||
|
||||
// isUserFilePopulated checks if USER.md has been filled with actual user data
|
||||
@@ -48,7 +47,7 @@ func (l *Loop) finalizeRun(
|
||||
// BEFORE sanitize strips them. Covers cases where a tool returned its
|
||||
// artifact via the ForLLM MEDIA: prefix but the agent relayed it as plain
|
||||
// text (e.g. PDF from exec/weasyprint, TTS mp3 paths the LLM quotes back).
|
||||
if extracted := extractMediaFromContent(rs.finalContent, tools.ToolWorkspaceFromCtx(ctx)); len(extracted) > 0 {
|
||||
if extracted := extractMediaFromContent(rs.finalContent, l.mediaEgressRoots(ctx)); len(extracted) > 0 {
|
||||
rs.mediaResults = append(rs.mediaResults, extracted...)
|
||||
}
|
||||
|
||||
@@ -67,8 +66,12 @@ func (l *Loop) finalizeRun(
|
||||
rs.finalContent += "\n\n---\n_" + i18n.T(locale, i18n.MsgSkillNudgePostscript) + "_"
|
||||
}
|
||||
|
||||
// 7. Fallback for empty content
|
||||
if rs.finalContent == "" {
|
||||
// 7. Fallback only when there is no other deliverable output. Media-only
|
||||
// runs must remain media-only instead of gaining a visible "..." caption.
|
||||
hasDeliverableOutput := len(rs.mediaResults) > 0 ||
|
||||
len(req.ForwardMedia) > 0 ||
|
||||
req.ContentSuffix != ""
|
||||
if rs.finalContent == "" && !hasDeliverableOutput {
|
||||
if len(rs.asyncToolCalls) > 0 {
|
||||
rs.finalContent = "..."
|
||||
} else {
|
||||
@@ -200,6 +203,9 @@ func (l *Loop) finalizeRun(
|
||||
slog.Info("agent loop: NO_REPLY detected, suppressing delivery",
|
||||
"agent", l.id, "session", req.SessionKey)
|
||||
rs.finalContent = ""
|
||||
if req.ContentSuffix != "" {
|
||||
rs.finalContent = deduplicateMediaSuffix("", req.ContentSuffix)
|
||||
}
|
||||
}
|
||||
|
||||
// 9. Maybe summarize
|
||||
|
||||
@@ -48,6 +48,14 @@ func (l *Loop) buildMessages(ctx context.Context, history []providers.Message, s
|
||||
tools.UserChatLayer(tools.SanitizePathSegment(userID), shared),
|
||||
)
|
||||
}
|
||||
if tools.IsDelegationArtifactRun(ctx) {
|
||||
promptWorkspace = "."
|
||||
const artifactGuidance = "Delegation workspace: write outputs using ordinary relative paths in the current workspace. Read staged inputs only through inputs/... . Files are returned to the caller only after runtime validation and publication."
|
||||
if extraSystemPrompt != "" {
|
||||
extraSystemPrompt += "\n\n"
|
||||
}
|
||||
extraSystemPrompt += artifactGuidance
|
||||
}
|
||||
|
||||
// Resolve context files once — also detect BOOTSTRAP.md presence.
|
||||
// lightContext: skip loading context files, only inject ExtraSystemPrompt (heartbeat checklist).
|
||||
|
||||
@@ -65,6 +65,9 @@ func (l *Loop) enrichInputMedia(ctx context.Context, req *RunRequest, messages [
|
||||
var mediaRefs []providers.MediaRef
|
||||
if len(req.Media) > 0 {
|
||||
mediaRefs = l.persistMedia(req.SessionKey, req.Media, tools.ToolWorkspaceFromCtx(ctx))
|
||||
if req.RunKind == "delegate" {
|
||||
rehomeDelegatedMediaMessage(messages, req.Media, mediaRefs, tools.ToolWorkspaceFromCtx(ctx))
|
||||
}
|
||||
|
||||
// Register persisted text uploads in vault (async, non-blocking).
|
||||
if l.onTextUploaded != nil {
|
||||
@@ -109,46 +112,55 @@ func (l *Loop) enrichInputMedia(ctx context.Context, req *RunRequest, messages [
|
||||
ctx = l.loadHistoricalImagesForTool(ctx, mediaRefs, messages)
|
||||
}
|
||||
|
||||
// 2b. Collect document MediaRefs (historical + current) for read_document tool.
|
||||
// 2b. Collect image MediaRefs (historical + current) for exact media_id
|
||||
// resolution by read_image and create_image.
|
||||
if imageRefs := collectRefsByKind(messages, mediaRefs, "image"); len(imageRefs) > 0 {
|
||||
ctx = tools.WithMediaImageRefs(ctx, imageRefs)
|
||||
}
|
||||
|
||||
// 2c. Collect document MediaRefs (historical + current) for read_document tool.
|
||||
if docRefs := collectRefsByKind(messages, mediaRefs, "document"); len(docRefs) > 0 {
|
||||
ctx = tools.WithMediaDocRefs(ctx, docRefs)
|
||||
// Enrich the last user message with persisted file paths so skills can access
|
||||
// documents via exec (e.g. pypdf). Only for current-turn refs (just persisted).
|
||||
l.enrichDocumentPaths(messages, mediaRefs)
|
||||
// Enrich the last user message with exact IDs and logical workspace paths.
|
||||
// Only current-turn refs are paired with current-turn tags.
|
||||
l.enrichDocumentPaths(messages, mediaRefs, tools.ToolWorkspaceFromCtx(ctx))
|
||||
}
|
||||
|
||||
// 2c. Collect audio MediaRefs (historical + current) for read_audio tool.
|
||||
// 2d. Collect audio MediaRefs (historical + current) for read_audio tool.
|
||||
if audioRefs := collectRefsByKind(messages, mediaRefs, "audio"); len(audioRefs) > 0 {
|
||||
ctx = tools.WithMediaAudioRefs(ctx, audioRefs)
|
||||
l.enrichAudioIDs(messages, mediaRefs)
|
||||
l.enrichAudioIDs(messages, mediaRefs, tools.ToolWorkspaceFromCtx(ctx))
|
||||
}
|
||||
|
||||
// 2d. Collect video MediaRefs (historical + current) for read_video tool.
|
||||
// 2e. Collect video MediaRefs (historical + current) for read_video tool.
|
||||
if videoRefs := collectRefsByKind(messages, mediaRefs, "video"); len(videoRefs) > 0 {
|
||||
ctx = tools.WithMediaVideoRefs(ctx, videoRefs)
|
||||
l.enrichVideoIDs(messages, mediaRefs)
|
||||
}
|
||||
|
||||
// 2e. Enrich <media:image> tags with persisted media IDs so the LLM
|
||||
// 2f. Enrich <media:image> tags with persisted media IDs so the LLM
|
||||
// knows images were received and stored (consistent with audio/video enrichment).
|
||||
l.enrichImageIDs(messages, mediaRefs)
|
||||
l.enrichImageIDs(messages, mediaRefs, tools.ToolWorkspaceFromCtx(ctx))
|
||||
|
||||
// 2e-ii. In file-ref mode, enrich ALL user messages' image tags with file paths.
|
||||
// This enables read_image(path=...) for both current and historical images.
|
||||
if deferToReadImageTool {
|
||||
l.enrichImagePaths(messages)
|
||||
}
|
||||
// 2f-ii. Enrich ALL user messages' image tags with logical file paths.
|
||||
// This upgrades legacy absolute paths and keeps current and historical media
|
||||
// references safe regardless of whether images are also attached inline.
|
||||
l.enrichImagePaths(messages, tools.ToolWorkspaceFromCtx(ctx))
|
||||
|
||||
// 2f. Collect all media file paths for team workspace auto-collect.
|
||||
// 2g. Collect all media file paths for team workspace auto-collect.
|
||||
// When the leader calls team_tasks(create), these paths are copied to the
|
||||
// team workspace so members can access attached files.
|
||||
if len(mediaRefs) > 0 && l.mediaStore != nil {
|
||||
if len(mediaRefs) > 0 {
|
||||
var mediaPaths []string
|
||||
for _, ref := range mediaRefs {
|
||||
// Prefer workspace-local path (.uploads/) over canonical .media/ path.
|
||||
if ref.Path != "" {
|
||||
mediaPaths = append(mediaPaths, ref.Path)
|
||||
} else if p, err := l.mediaStore.LoadPath(ref.ID); err == nil {
|
||||
} else if l.mediaStore != nil {
|
||||
p, err := l.mediaStore.LoadPath(ref.ID)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
mediaPaths = append(mediaPaths, p)
|
||||
}
|
||||
}
|
||||
@@ -166,3 +178,70 @@ func (l *Loop) enrichInputMedia(ctx context.Context, req *RunRequest, messages [
|
||||
|
||||
return ctx, messages, mediaRefs
|
||||
}
|
||||
|
||||
// rehomeDelegatedMediaMessage replaces caller-owned attachment paths with the
|
||||
// delegatee-owned persisted copies and ensures every imported attachment has a
|
||||
// media tag. The normal enrichment pass then adds the new IDs and local paths.
|
||||
func rehomeDelegatedMediaMessage(
|
||||
messages []providers.Message,
|
||||
input []bus.MediaFile,
|
||||
refs []providers.MediaRef,
|
||||
workspace string,
|
||||
) {
|
||||
if len(messages) == 0 || len(refs) == 0 {
|
||||
return
|
||||
}
|
||||
lastUser := -1
|
||||
for i := len(messages) - 1; i >= 0; i-- {
|
||||
if messages[i].Role == "user" {
|
||||
lastUser = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if lastUser < 0 {
|
||||
return
|
||||
}
|
||||
|
||||
content := messages[lastUser].Content
|
||||
for i := 0; i < len(input) && i < len(refs); i++ {
|
||||
if input[i].Path != "" && refs[i].Path != "" {
|
||||
replacement := logicalWorkspaceMediaPath(workspace, refs[i].Path)
|
||||
if replacement == "" {
|
||||
replacement = filepath.Base(refs[i].Path)
|
||||
}
|
||||
content = strings.ReplaceAll(content, input[i].Path, replacement)
|
||||
}
|
||||
}
|
||||
|
||||
tagCounts := map[string]int{
|
||||
"image": strings.Count(content, "<media:image"),
|
||||
"document": strings.Count(content, "<media:document"),
|
||||
"audio": strings.Count(content, "<media:audio") + strings.Count(content, "<media:voice"),
|
||||
"video": strings.Count(content, "<media:video"),
|
||||
}
|
||||
for _, ref := range refs {
|
||||
tag := ""
|
||||
switch ref.Kind {
|
||||
case "image":
|
||||
tag = "<media:image>"
|
||||
case "document":
|
||||
tag = "<media:document>"
|
||||
case "audio":
|
||||
tag = "<media:audio>"
|
||||
case "video":
|
||||
tag = "<media:video>"
|
||||
}
|
||||
if tag == "" {
|
||||
continue
|
||||
}
|
||||
if tagCounts[ref.Kind] > 0 {
|
||||
tagCounts[ref.Kind]--
|
||||
continue
|
||||
}
|
||||
if content != "" && !strings.HasSuffix(content, "\n") {
|
||||
content += "\n"
|
||||
}
|
||||
content += tag
|
||||
}
|
||||
messages[lastUser].Content = content
|
||||
}
|
||||
@@ -1,9 +1,12 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/tools"
|
||||
)
|
||||
|
||||
// parseMediaResult extracts a MediaResult from a tool result string containing "MEDIA:" prefix.
|
||||
@@ -94,8 +97,7 @@ func confineToWorkspace(mediaPath, workspace string) (string, bool) {
|
||||
path = filepath.Join(wsRoot, path)
|
||||
}
|
||||
cleaned := filepath.Clean(path)
|
||||
info, err := os.Lstat(cleaned)
|
||||
if err != nil || !info.Mode().IsRegular() {
|
||||
if err := tools.ValidateRegularFileForRead(cleaned); err != nil {
|
||||
return "", false
|
||||
}
|
||||
resolved, err := filepath.EvalSymlinks(cleaned)
|
||||
@@ -109,18 +111,31 @@ func confineToWorkspace(mediaPath, workspace string) (string, bool) {
|
||||
return cleaned, true
|
||||
}
|
||||
|
||||
// confineToAnyRoot accepts mediaPath if it is contained by ANY of the allowed
|
||||
// roots (each checked with the hardened confineToWorkspace). Used for the
|
||||
// result.Media egress: a tool's media legitimately lives in the agent
|
||||
// workspace, the team workspace, OR a tenant-allowed path — the same scopes the
|
||||
// producing tools (create_*, send_file, delegate) validate against. A path
|
||||
// outside every root (e.g. /etc/passwd from a prompt-injected path) is rejected,
|
||||
// so the egress guard holds without dropping legitimate cross-workspace media.
|
||||
// confineToAnyRoot accepts an absolute mediaPath if it is contained by any
|
||||
// allowed root. Relative paths are intentionally anchored to the first root,
|
||||
// which is always the active tool workspace; extra roots authorize explicit
|
||||
// paths but never change the meaning of an ambiguous relative path.
|
||||
func confineToAnyRoot(mediaPath string, roots []string) (string, bool) {
|
||||
for _, root := range roots {
|
||||
if len(roots) == 0 {
|
||||
return "", false
|
||||
}
|
||||
if !filepath.IsAbs(mediaPath) {
|
||||
return confineToWorkspace(mediaPath, roots[0])
|
||||
}
|
||||
for i, root := range roots {
|
||||
if root == "" {
|
||||
continue
|
||||
}
|
||||
// The active workspace (index 0) may itself be reached through an
|
||||
// operator-managed symlink. Extra read roots are authorization
|
||||
// boundaries and must be real directories, not partner-controlled
|
||||
// symlink aliases to arbitrary locations.
|
||||
if i > 0 {
|
||||
if info, err := os.Lstat(root); err == nil &&
|
||||
(info.Mode()&os.ModeSymlink != 0 || !info.IsDir()) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if cleaned, ok := confineToWorkspace(mediaPath, root); ok {
|
||||
return cleaned, true
|
||||
}
|
||||
@@ -128,17 +143,43 @@ func confineToAnyRoot(mediaPath string, roots []string) (string, bool) {
|
||||
return "", false
|
||||
}
|
||||
|
||||
// mediaEgressRoots returns every read-authorized root that may supply outbound
|
||||
// media. The active workspace stays first to preserve relative path semantics.
|
||||
func (l *Loop) mediaEgressRoots(ctx context.Context) []string {
|
||||
tenantAllowedPaths := tools.TenantAllowedPathsFromCtx(ctx)
|
||||
candidates := make([]string, 0, 3+len(tenantAllowedPaths))
|
||||
candidates = append(candidates,
|
||||
tools.ToolWorkspaceFromCtx(ctx),
|
||||
tools.ToolTeamWorkspaceFromCtx(ctx),
|
||||
tools.ToolTeamRootFromCtx(ctx),
|
||||
)
|
||||
candidates = append(candidates, tenantAllowedPaths...)
|
||||
|
||||
seen := make(map[string]struct{}, len(candidates))
|
||||
roots := make([]string, 0, len(candidates))
|
||||
for _, root := range candidates {
|
||||
if root == "" {
|
||||
continue
|
||||
}
|
||||
cleaned := filepath.Clean(root)
|
||||
if _, ok := seen[cleaned]; ok {
|
||||
continue
|
||||
}
|
||||
seen[cleaned] = struct{}{}
|
||||
roots = append(roots, cleaned)
|
||||
}
|
||||
return roots
|
||||
}
|
||||
|
||||
// extractMediaFromContent scans text for MEDIA:<path> tokens the LLM may echo
|
||||
// in its final response (e.g. when a tool returned the MEDIA: prefix as plain
|
||||
// text instead of setting Result.Media). Relative paths are resolved against
|
||||
// workspace. Called before sanitize strips the tokens so the attachments are
|
||||
// still delivered.
|
||||
// text instead of setting Result.Media). The first root is the active workspace
|
||||
// used for relative paths; later roots authorize explicit absolute paths.
|
||||
// Called before sanitize strips the tokens so the attachments are delivered.
|
||||
//
|
||||
// Security: only paths accepted by confineToWorkspace are emitted. An LLM cannot
|
||||
// inject attachments pointing at /etc/passwd, a sibling tenant's workspace, or a
|
||||
// hallucinated path — the extractor silently drops them.
|
||||
func extractMediaFromContent(content, workspace string) []MediaResult {
|
||||
if !strings.Contains(content, "MEDIA:") || workspace == "" {
|
||||
// Security: only paths accepted by confineToAnyRoot are emitted.
|
||||
func extractMediaFromContent(content string, roots []string) []MediaResult {
|
||||
if !strings.Contains(content, "MEDIA:") || len(roots) == 0 {
|
||||
return nil
|
||||
}
|
||||
matches := mediaPathPattern.FindAllString(content, -1)
|
||||
@@ -158,7 +199,7 @@ func extractMediaFromContent(content, workspace string) []MediaResult {
|
||||
if path == "" {
|
||||
continue
|
||||
}
|
||||
cleaned, ok := confineToWorkspace(path, workspace)
|
||||
cleaned, ok := confineToAnyRoot(path, roots)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,9 +1,13 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/tools"
|
||||
)
|
||||
|
||||
// writeTempFile drops a zero-byte file at workspace/relPath, creating dirs.
|
||||
@@ -31,6 +35,12 @@ func TestExtractMediaFromContent(t *testing.T) {
|
||||
audioA := writeTempFile(t, ws, "a.mp3")
|
||||
audioB := writeTempFile(t, ws, "b.mp3")
|
||||
chartPath := writeTempFile(t, ws, "charts/q4.png")
|
||||
partnerRaw := t.TempDir()
|
||||
partner, err := filepath.EvalSymlinks(partnerRaw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
partnerPath := writeTempFile(t, partner, "partner-result.png")
|
||||
|
||||
// Outside-workspace file: should be rejected by containment check.
|
||||
outsideDir := t.TempDir()
|
||||
@@ -38,6 +48,14 @@ func TestExtractMediaFromContent(t *testing.T) {
|
||||
if err := os.WriteFile(outsidePath, nil, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hardlinkPath := filepath.Join(partner, "hardlink-result.png")
|
||||
if err := os.Link(outsidePath, hardlinkPath); err != nil {
|
||||
t.Skipf("hardlinks not supported: %v", err)
|
||||
}
|
||||
symlinkRoot := filepath.Join(t.TempDir(), "delegate")
|
||||
if err := os.Symlink(outsideDir, symlinkRoot); err != nil {
|
||||
t.Skipf("symlinks not supported: %v", err)
|
||||
}
|
||||
|
||||
// Symlink inside workspace pointing to outside: must be rejected by
|
||||
// EvalSymlinks-then-Rel containment. Covers the P0 ancestor-symlink
|
||||
@@ -59,7 +77,7 @@ func TestExtractMediaFromContent(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
content string
|
||||
workspace string
|
||||
roots []string
|
||||
wantPaths []string
|
||||
}{
|
||||
{
|
||||
@@ -73,54 +91,75 @@ func TestExtractMediaFromContent(t *testing.T) {
|
||||
{
|
||||
name: "relative path resolved + exists",
|
||||
content: "MEDIA:deliver/report.pdf",
|
||||
workspace: ws,
|
||||
roots: []string{ws},
|
||||
wantPaths: []string{reportPath},
|
||||
},
|
||||
{
|
||||
name: "multiple tokens deduped",
|
||||
content: "First: MEDIA:a.mp3\nSecond: MEDIA:b.mp3\nAgain: MEDIA:a.mp3",
|
||||
workspace: ws,
|
||||
roots: []string{ws},
|
||||
wantPaths: []string{audioA, audioB},
|
||||
},
|
||||
{
|
||||
name: "markdown wrapped and punctuation stripped",
|
||||
content: `. See "MEDIA:deliver/report.pdf".`,
|
||||
workspace: ws,
|
||||
roots: []string{ws},
|
||||
wantPaths: []string{chartPath, reportPath},
|
||||
},
|
||||
{
|
||||
name: "hallucinated path dropped (file missing)",
|
||||
content: "MEDIA:not-real.pdf",
|
||||
workspace: ws,
|
||||
name: "hallucinated path dropped (file missing)",
|
||||
content: "MEDIA:not-real.pdf",
|
||||
roots: []string{ws},
|
||||
},
|
||||
{
|
||||
name: "path traversal escape blocked",
|
||||
content: "MEDIA:../leak.pdf",
|
||||
workspace: ws,
|
||||
name: "path traversal escape blocked",
|
||||
content: "MEDIA:../leak.pdf",
|
||||
roots: []string{ws},
|
||||
},
|
||||
{
|
||||
name: "absolute path outside workspace blocked",
|
||||
content: "MEDIA:" + outsidePath,
|
||||
workspace: ws,
|
||||
name: "absolute path outside workspace blocked",
|
||||
content: "MEDIA:" + outsidePath,
|
||||
roots: []string{ws},
|
||||
},
|
||||
{
|
||||
name: "absolute collaboration path allowed",
|
||||
content: "MEDIA:" + partnerPath,
|
||||
roots: []string{ws, partner},
|
||||
wantPaths: []string{partnerPath},
|
||||
},
|
||||
{
|
||||
name: "hardlink in collaboration path blocked",
|
||||
content: "MEDIA:" + hardlinkPath,
|
||||
roots: []string{ws, partner},
|
||||
},
|
||||
{
|
||||
name: "symlinked collaboration root blocked",
|
||||
content: "MEDIA:" + outsidePath,
|
||||
roots: []string{ws, symlinkRoot},
|
||||
},
|
||||
{
|
||||
name: "relative path does not search collaboration roots",
|
||||
content: "MEDIA:partner-result.png",
|
||||
roots: []string{ws, partner},
|
||||
},
|
||||
{
|
||||
name: "absolute path with no workspace dropped",
|
||||
content: "MEDIA:" + reportPath,
|
||||
},
|
||||
{
|
||||
name: "symlink leaf rejected by Lstat",
|
||||
content: "MEDIA:shortcut-to-leak.pdf",
|
||||
workspace: ws,
|
||||
name: "symlink leaf rejected by Lstat",
|
||||
content: "MEDIA:shortcut-to-leak.pdf",
|
||||
roots: []string{ws},
|
||||
},
|
||||
{
|
||||
name: "ancestor symlink escape blocked (P0)",
|
||||
content: "MEDIA:shared/victim.pdf",
|
||||
workspace: ws,
|
||||
name: "ancestor symlink escape blocked (P0)",
|
||||
content: "MEDIA:shared/victim.pdf",
|
||||
roots: []string{ws},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := extractMediaFromContent(tt.content, tt.workspace)
|
||||
got := extractMediaFromContent(tt.content, tt.roots)
|
||||
if len(got) != len(tt.wantPaths) {
|
||||
t.Fatalf("count = %d, want %d; got=%+v", len(got), len(tt.wantPaths), got)
|
||||
}
|
||||
@@ -133,6 +172,25 @@ func TestExtractMediaFromContent(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMediaEgressRoots(t *testing.T) {
|
||||
ctx := tools.WithToolWorkspace(context.Background(), "/workspace/parent")
|
||||
ctx = tools.WithToolTeamWorkspace(ctx, "/workspace/team/chat")
|
||||
ctx = tools.WithToolTeamRoot(ctx, "/workspace/team")
|
||||
ctx = tools.WithTenantAllowedPaths(ctx, []string{"/workspace/tenant-export"})
|
||||
loop := NewLoop(LoopConfig{})
|
||||
|
||||
got := loop.mediaEgressRoots(ctx)
|
||||
want := []string{
|
||||
"/workspace/parent",
|
||||
"/workspace/team/chat",
|
||||
"/workspace/team",
|
||||
"/workspace/tenant-export",
|
||||
}
|
||||
if !slices.Equal(got, want) {
|
||||
t.Fatalf("mediaEgressRoots() = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConfineToWorkspace exercises the shared media path-containment boundary
|
||||
// directly. It is the single guard that both feeders of MediaResult.Path rely
|
||||
// on, so a regression here would reopen the outbound-exfiltration hole (H2).
|
||||
@@ -180,6 +238,7 @@ func TestConfineToWorkspace(t *testing.T) {
|
||||
{name: "absolute outside workspace rejected", path: outsidePath, workspace: ws, wantOK: false},
|
||||
{name: "traversal escape rejected", path: "../secret.txt", workspace: ws, wantOK: false},
|
||||
{name: "missing file rejected", path: "nope.pdf", workspace: ws, wantOK: false},
|
||||
{name: "directory rejected", path: "deliver", workspace: ws, wantOK: false},
|
||||
{name: "empty workspace rejected", path: insidePath, workspace: "", wantOK: false},
|
||||
{name: "empty path rejected", path: "", workspace: ws, wantOK: false},
|
||||
{name: "leaf symlink rejected", path: "shortcut.txt", workspace: ws, wantOK: false, symlink: true},
|
||||
|
||||
@@ -45,7 +45,7 @@ func (l *Loop) runViaPipeline(ctx context.Context, req RunRequest) (*RunResult,
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return convertRunResult(pResult), nil
|
||||
return redactDelegationRunResult(&req, convertRunResult(pResult)), nil
|
||||
}
|
||||
|
||||
// buildPipelineDeps maps Loop fields + methods to PipelineDeps callbacks.
|
||||
@@ -94,7 +94,7 @@ func (l *Loop) buildPipelineDeps(req *RunRequest, bridgeRS *runState) pipeline.P
|
||||
ResolveContextWindow: l.resolveEffectiveContextWindow,
|
||||
EmitEvent: func(event any) {
|
||||
if ae, ok := event.(AgentEvent); ok {
|
||||
l.emit(ae)
|
||||
l.emit(redactDelegationAgentEvent(req, ae))
|
||||
}
|
||||
},
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@ func (l *Loop) pipelineCallbacks(req *RunRequest, bridgeRS *runState) pipelineCa
|
||||
event.ChatID = req.ChatID
|
||||
event.SessionKey = req.SessionKey
|
||||
event.TenantID = l.tenantID
|
||||
l.emit(event)
|
||||
l.emit(redactDelegationAgentEvent(req, event))
|
||||
}
|
||||
return pipelineCallbackSet{
|
||||
emitRun: emitRun,
|
||||
@@ -188,7 +188,7 @@ func (l *Loop) makeEnrichMedia(req *RunRequest) func(ctx context.Context, state
|
||||
if len(msgs) == 0 {
|
||||
return nil
|
||||
}
|
||||
enrichedCtx, enrichedMsgs, _ := l.enrichInputMedia(ctx, req, msgs)
|
||||
enrichedCtx, enrichedMsgs, currentRefs := l.enrichInputMedia(ctx, req, msgs)
|
||||
// Propagate enriched context (media images/docs/audio/video refs for tools).
|
||||
state.Ctx = enrichedCtx
|
||||
// Update history with enriched messages (media tags, inline images).
|
||||
@@ -196,6 +196,23 @@ func (l *Loop) makeEnrichMedia(req *RunRequest) func(ctx context.Context, state
|
||||
if len(enrichedMsgs) > 1 {
|
||||
state.Messages.SetHistory(enrichedMsgs[1:])
|
||||
}
|
||||
// Preserve the enriched current input for the first session checkpoint.
|
||||
// Inline image bytes stay request-local; durable history stores only the
|
||||
// logical tags plus absolute MediaRefs used internally for exact lookup.
|
||||
if len(currentRefs) > 0 {
|
||||
for i := len(enrichedMsgs) - 1; i >= 0; i-- {
|
||||
if enrichedMsgs[i].Role != "user" {
|
||||
continue
|
||||
}
|
||||
req.enrichedInputMessage = providers.Message{
|
||||
Role: "user",
|
||||
Content: enrichedMsgs[i].Content,
|
||||
MediaRefs: append([]providers.MediaRef(nil), currentRefs...),
|
||||
}
|
||||
req.hasEnrichedInputMessage = true
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -365,6 +382,12 @@ func (l *Loop) makeCallLLM(req *RunRequest, emitRun func(AgentEvent)) func(ctx c
|
||||
chatReq.Options[providers.OptPeerKind] = req.PeerKind
|
||||
chatReq.Options[providers.OptLocalKey] = req.LocalKey
|
||||
chatReq.Options[providers.OptWorkspace] = tools.ToolWorkspaceFromCtx(ctx)
|
||||
if delegationID := tools.DelegationIDFromCtx(ctx); delegationID != "" {
|
||||
chatReq.Options[providers.OptDelegationID] = delegationID
|
||||
}
|
||||
if inputs := tools.DelegationArtifactInputsFromCtx(ctx); inputs != "" {
|
||||
chatReq.Options[providers.OptDelegationInputs] = inputs
|
||||
}
|
||||
// Pass the policy-filtered allowed tool set so the Claude CLI provider
|
||||
// can restrict its native built-in tools (Bash, Edit, Write, Read,
|
||||
// WebFetch, WebSearch) to what the agent's tool policy actually allows.
|
||||
@@ -706,13 +729,17 @@ func (l *Loop) makeFlushMessages(req *RunRequest) func(ctx context.Context, sess
|
||||
return func(ctx context.Context, sessionKey string, msgs []providers.Message) error {
|
||||
if !userMsgFlushed && !req.HideInput && req.Message != "" {
|
||||
userMsgFlushed = true
|
||||
l.sessions.AddMessage(ctx, sessionKey, providers.Message{
|
||||
inputMessage := providers.Message{
|
||||
Role: "user",
|
||||
Content: req.Message,
|
||||
})
|
||||
}
|
||||
if req.hasEnrichedInputMessage {
|
||||
inputMessage = req.enrichedInputMessage
|
||||
}
|
||||
l.sessions.AddMessage(ctx, sessionKey, redactDelegationMessage(req, inputMessage))
|
||||
}
|
||||
for _, msg := range msgs {
|
||||
l.sessions.AddMessage(ctx, sessionKey, msg)
|
||||
l.sessions.AddMessage(ctx, sessionKey, redactDelegationMessage(req, msg))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -2,13 +2,17 @@ package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/bus"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/pipeline"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/providers"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/tools"
|
||||
"github.com/nextlevelbuilder/goclaw/pkg/protocol"
|
||||
)
|
||||
|
||||
@@ -25,6 +29,32 @@ func (p finalThinkingStreamProvider) ChatStream(context.Context, providers.ChatR
|
||||
func (p finalThinkingStreamProvider) DefaultModel() string { return "test-model" }
|
||||
func (p finalThinkingStreamProvider) Name() string { return "test-provider" }
|
||||
|
||||
type requestCaptureProvider struct {
|
||||
request providers.ChatRequest
|
||||
}
|
||||
|
||||
func (p *requestCaptureProvider) Chat(_ context.Context, req providers.ChatRequest) (*providers.ChatResponse, error) {
|
||||
p.request = req
|
||||
return &providers.ChatResponse{Content: "ok"}, nil
|
||||
}
|
||||
|
||||
func (p *requestCaptureProvider) ChatStream(_ context.Context, req providers.ChatRequest, _ func(providers.StreamChunk)) (*providers.ChatResponse, error) {
|
||||
p.request = req
|
||||
return &providers.ChatResponse{Content: "ok"}, nil
|
||||
}
|
||||
|
||||
func (p *requestCaptureProvider) DefaultModel() string { return "test-model" }
|
||||
func (p *requestCaptureProvider) Name() string { return "test-provider" }
|
||||
|
||||
type recordingSessionStore struct {
|
||||
*nopSessionStore
|
||||
added []providers.Message
|
||||
}
|
||||
|
||||
func (s *recordingSessionStore) AddMessage(_ context.Context, _ string, msg providers.Message) {
|
||||
s.added = append(s.added, msg)
|
||||
}
|
||||
|
||||
func TestMakeCallLLM_StreamsFinalThinkingWhenNoThinkingChunkArrives(t *testing.T) {
|
||||
col := &eventCollector{}
|
||||
loop := &Loop{id: "test-agent", onEvent: col.onEvent}
|
||||
@@ -58,6 +88,73 @@ func TestMakeCallLLM_StreamsFinalThinkingWhenNoThinkingChunkArrives(t *testing.T
|
||||
}
|
||||
}
|
||||
|
||||
func TestMakeCallLLMPropagatesDelegationArtifactBridgeOptions(t *testing.T) {
|
||||
provider := &requestCaptureProvider{}
|
||||
loop := &Loop{id: "target-agent", agentUUID: uuid.New()}
|
||||
req := &RunRequest{RunID: "run-1", SessionKey: "delegate:session"}
|
||||
state := &pipeline.RunState{Provider: provider, Model: "test-model"}
|
||||
ctx := tools.WithToolWorkspace(context.Background(), "/runtime/outputs")
|
||||
ctx = tools.WithDelegationID(ctx, "delegation-id")
|
||||
ctx = tools.WithDelegationArtifactInputs(ctx, "/runtime/inputs")
|
||||
|
||||
if _, err := loop.makeCallLLM(req, func(AgentEvent) {})(ctx, state, providers.ChatRequest{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if provider.request.Options[providers.OptDelegationID] != "delegation-id" ||
|
||||
provider.request.Options[providers.OptDelegationInputs] != "/runtime/inputs" {
|
||||
t.Fatalf("delegation options = %#v", provider.request.Options)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnrichedInputMediaPersistsForNextTurn(t *testing.T) {
|
||||
sourceDir := t.TempDir()
|
||||
sourcePath := filepath.Join(sourceDir, "photo.png")
|
||||
if err := os.WriteFile(sourcePath, minimalPNG, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
workspace := t.TempDir()
|
||||
sessions := &recordingSessionStore{nopSessionStore: &nopSessionStore{}}
|
||||
loop := &Loop{sessions: sessions}
|
||||
req := &RunRequest{
|
||||
SessionKey: "session-media",
|
||||
Message: `<media:image url="attachment://photo.png">`,
|
||||
Media: []bus.MediaFile{{
|
||||
Path: sourcePath,
|
||||
MimeType: "image/png",
|
||||
Filename: "photo.png",
|
||||
}},
|
||||
}
|
||||
state := &pipeline.RunState{Messages: pipeline.NewMessageBuffer(providers.Message{Role: "system"})}
|
||||
state.Messages.SetHistory([]providers.Message{{Role: "user", Content: req.Message}})
|
||||
ctx := tools.WithToolWorkspace(context.Background(), workspace)
|
||||
|
||||
if err := loop.makeEnrichMedia(req)(ctx, state); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := loop.makeFlushMessages(req)(ctx, req.SessionKey, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(sessions.added) != 1 {
|
||||
t.Fatalf("persisted messages = %d, want 1", len(sessions.added))
|
||||
}
|
||||
persisted := sessions.added[0]
|
||||
if len(persisted.MediaRefs) != 1 {
|
||||
t.Fatalf("persisted MediaRefs = %#v, want one image ref", persisted.MediaRefs)
|
||||
}
|
||||
if strings.Contains(persisted.Content, workspace) {
|
||||
t.Fatalf("persisted content leaked workspace path: %q", persisted.Content)
|
||||
}
|
||||
if !strings.Contains(persisted.Content, `path=".uploads/`) {
|
||||
t.Fatalf("persisted content lacks logical image path: %q", persisted.Content)
|
||||
}
|
||||
|
||||
nextTurnRefs := collectRefsByKind([]providers.Message{persisted}, nil, "image")
|
||||
if len(nextTurnRefs) != 1 || nextTurnRefs[0].ID != persisted.MediaRefs[0].ID {
|
||||
t.Fatalf("next-turn refs = %#v, want persisted exact ID", nextTurnRefs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptCacheOptionsHelpers(t *testing.T) {
|
||||
tenantID := uuid.MustParse("11111111-1111-1111-1111-111111111111")
|
||||
agentID := uuid.MustParse("22222222-2222-2222-2222-222222222222")
|
||||
|
||||
@@ -384,6 +384,6 @@ func makeToolEmitRun(l *Loop, req *RunRequest) func(AgentEvent) {
|
||||
event.Channel = req.Channel
|
||||
event.ChatID = req.ChatID
|
||||
event.TenantID = l.tenantID
|
||||
l.emit(event)
|
||||
l.emit(redactDelegationAgentEvent(req, event))
|
||||
}
|
||||
}
|
||||
@@ -2,9 +2,11 @@ package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/bus"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/config"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/pipeline"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/providers"
|
||||
@@ -72,6 +74,44 @@ func newTestLoopForToolCallbacks(onEvent func(AgentEvent)) *Loop {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessToolResultSuppressesUnpublishedArtifactMediaBeforeEvent(t *testing.T) {
|
||||
col := &eventCollector{}
|
||||
loop := newTestLoopForToolCallbacks(col.onEvent)
|
||||
req := &RunRequest{RunID: "run-artifact"}
|
||||
ctx := tools.WithDelegationID(context.Background(), "delegation-1")
|
||||
ctx = tools.WithDelegationArtifactInputs(ctx, "/runtime/delegations/delegation-1/inputs")
|
||||
result := &tools.Result{
|
||||
ForLLM: "created image\nMEDIA:/runtime/delegations/delegation-1/outputs/image.png",
|
||||
Media: []bus.MediaFile{{Path: "/runtime/delegations/delegation-1/outputs/image.png"}},
|
||||
}
|
||||
|
||||
toolMsg, _, _ := loop.processToolResult(
|
||||
ctx,
|
||||
&runState{},
|
||||
req,
|
||||
col.onEvent,
|
||||
providers.ToolCall{ID: "tc-artifact", Name: "create_image"},
|
||||
"create_image",
|
||||
result,
|
||||
false,
|
||||
)
|
||||
|
||||
if strings.Contains(toolMsg.Content, "MEDIA:") || len(result.Media) != 0 {
|
||||
t.Fatalf("unpublished media survived result policy: message=%q media=%#v", toolMsg.Content, result.Media)
|
||||
}
|
||||
events := col.filter(protocol.AgentEventToolResult)
|
||||
if len(events) != 1 {
|
||||
t.Fatalf("tool result events = %d, want 1", len(events))
|
||||
}
|
||||
payload, ok := events[0].Payload.(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("event payload type = %T", events[0].Payload)
|
||||
}
|
||||
if value, _ := payload["result"].(string); strings.Contains(value, "MEDIA:") {
|
||||
t.Fatalf("event leaked unpublished media marker: %q", value)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMakeExecuteToolCall_EmitsToolCallEvent verifies the sequential wrapper
|
||||
// emits a tool.call event before running tool I/O.
|
||||
func TestMakeExecuteToolCall_EmitsToolCallEvent(t *testing.T) {
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
func (l *Loop) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
l.activeRuns.Add(1)
|
||||
defer l.activeRuns.Add(-1)
|
||||
ctx = withDelegationArtifactTextRedactor(ctx, &req)
|
||||
|
||||
// Per-run emit wrapper: enriches every AgentEvent with delegation + routing context.
|
||||
emitRun := func(event AgentEvent) {
|
||||
@@ -32,7 +33,7 @@ func (l *Loop) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
event.ChatID = req.ChatID
|
||||
event.SessionKey = req.SessionKey
|
||||
event.TenantID = store.TenantIDFromContext(ctx)
|
||||
l.emit(event)
|
||||
l.emit(redactDelegationAgentEvent(&req, event))
|
||||
}
|
||||
|
||||
emitRun(AgentEvent{
|
||||
@@ -75,7 +76,7 @@ func (l *Loop) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
UserID: req.UserID,
|
||||
Channel: req.Channel,
|
||||
Name: traceName,
|
||||
InputPreview: truncateStr(req.Message, l.traceCollector.PreviewMaxLen()),
|
||||
InputPreview: tracing.RedactText(ctx, truncateStr(req.Message, l.traceCollector.PreviewMaxLen())),
|
||||
Status: store.TraceStatusRunning,
|
||||
StartTime: now,
|
||||
CreatedAt: now,
|
||||
@@ -140,7 +141,7 @@ func (l *Loop) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
l.emitAgentSpanEnd(safeCtx, agentSpanID, runStart, nil, context.Canceled)
|
||||
}
|
||||
l.traceCollector.FinishTrace(safeCtx, traceID, store.TraceStatusError,
|
||||
"trace finalized by safety net (likely panic or goroutine leak)", "")
|
||||
tracing.RedactText(safeCtx, "trace finalized by safety net (likely panic or goroutine leak)"), "")
|
||||
}()
|
||||
}
|
||||
|
||||
@@ -195,7 +196,7 @@ func (l *Loop) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
traceCtx = context.WithoutCancel(ctx)
|
||||
traceStatus = store.TraceStatusCancelled
|
||||
}
|
||||
l.traceCollector.FinishTrace(traceCtx, traceID, traceStatus, err.Error(), "")
|
||||
l.traceCollector.FinishTrace(traceCtx, traceID, traceStatus, tracing.RedactText(traceCtx, err.Error()), "")
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
@@ -243,7 +244,8 @@ func (l *Loop) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
if !isChildTrace && l.traceCollector != nil && traceID != uuid.Nil {
|
||||
traceFinalized = true
|
||||
if result != nil {
|
||||
l.traceCollector.FinishTrace(ctx, traceID, store.TraceStatusCompleted, "", truncateStr(result.Content, l.traceCollector.PreviewMaxLen()))
|
||||
l.traceCollector.FinishTrace(ctx, traceID, store.TraceStatusCompleted, "",
|
||||
tracing.RedactText(ctx, truncateStr(result.Content, l.traceCollector.PreviewMaxLen())))
|
||||
} else {
|
||||
l.traceCollector.FinishTrace(ctx, traceID, store.TraceStatusCompleted, "", "")
|
||||
}
|
||||
|
||||
@@ -36,6 +36,12 @@ func (l *Loop) processToolResult(
|
||||
hadBootstrap bool,
|
||||
) (toolMsg providers.Message, warningMsgs []providers.Message, action toolResultAction) {
|
||||
|
||||
// Agent Link outputs are not valid outbound media until the runtime has
|
||||
// validated and atomically published the delegation manifest. Apply this
|
||||
// before logs, events, loop detection, or message construction so no
|
||||
// pre-publication MEDIA marker escapes through observability surfaces.
|
||||
tools.ApplyDelegationArtifactResultPolicy(ctx, result)
|
||||
|
||||
// Record for loop detection.
|
||||
argsHash := rs.loopDetector.record(registryName, tc.Arguments)
|
||||
rs.loopDetector.recordResult(argsHash, result.ForLLM)
|
||||
@@ -85,6 +91,7 @@ func (l *Loop) processToolResult(
|
||||
|
||||
// Collect MEDIA: paths from tool results.
|
||||
// Prefer result.Media (explicit) over ForLLM MEDIA: prefix (legacy) to avoid duplicates.
|
||||
mediaRoots := l.mediaEgressRoots(ctx)
|
||||
if len(result.Media) > 0 {
|
||||
// Egress containment: a tool that sets result.Media[].Path to a path
|
||||
// outside every allowed scope (e.g. /etc/passwd from a prompt-injected
|
||||
@@ -94,10 +101,6 @@ func (l *Loop) processToolResult(
|
||||
// to — agent workspace, team workspace, and tenant-allowed paths — so a
|
||||
// cross-workspace file (e.g. a teammate-produced file in the shared team
|
||||
// workspace, or a synchronous delegatee's output) is not wrongly dropped.
|
||||
mediaRoots := append([]string{
|
||||
tools.ToolWorkspaceFromCtx(ctx),
|
||||
tools.ToolTeamWorkspaceFromCtx(ctx),
|
||||
}, l.tenantAllowedPaths...)
|
||||
for i, mf := range result.Media {
|
||||
cleaned, ok := confineToAnyRoot(mf.Path, mediaRoots)
|
||||
if !ok {
|
||||
@@ -123,7 +126,7 @@ func (l *Loop) processToolResult(
|
||||
// Telegram sendDocument). A malicious or buggy tool emitting
|
||||
// MEDIA:/etc/passwd is dropped here — fixing every channel at the source
|
||||
// rather than per-channel. Mirrors extractMediaFromContent containment.
|
||||
if cleaned, ok := confineToWorkspace(mr.Path, tools.ToolWorkspaceFromCtx(ctx)); ok {
|
||||
if cleaned, ok := confineToAnyRoot(mr.Path, mediaRoots); ok {
|
||||
mr.Path = cleaned
|
||||
rs.mediaResults = append(rs.mediaResults, *mr)
|
||||
} else {
|
||||
|
||||
@@ -152,7 +152,7 @@ func (l *Loop) emitLLMSpanStart(ctx context.Context, start time.Time, iteration
|
||||
}
|
||||
}
|
||||
|
||||
collector.EmitSpan(span)
|
||||
collector.EmitSpan(tracing.RedactSpan(ctx, span))
|
||||
return spanID
|
||||
}
|
||||
|
||||
@@ -244,7 +244,7 @@ func (l *Loop) emitLLMSpanEnd(ctx context.Context, spanID uuid.UUID, start time.
|
||||
updates["metadata"] = spanMetadata
|
||||
}
|
||||
|
||||
collector.EmitSpanUpdate(spanID, traceID, updates)
|
||||
collector.EmitSpanUpdate(spanID, traceID, tracing.RedactSpanUpdates(ctx, updates))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -289,7 +289,7 @@ func (l *Loop) emitToolSpanStart(ctx context.Context, start time.Time, toolName,
|
||||
span.TenantID = store.MasterTenantID
|
||||
}
|
||||
|
||||
collector.EmitSpan(span)
|
||||
collector.EmitSpan(tracing.RedactSpan(ctx, span))
|
||||
return spanID
|
||||
}
|
||||
|
||||
@@ -353,7 +353,7 @@ func (l *Loop) emitToolSpanEnd(ctx context.Context, spanID uuid.UUID, start time
|
||||
}
|
||||
}
|
||||
|
||||
collector.EmitSpanUpdate(spanID, traceID, updates)
|
||||
collector.EmitSpanUpdate(spanID, traceID, tracing.RedactSpanUpdates(ctx, updates))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -401,7 +401,7 @@ func (l *Loop) emitAgentSpanStart(ctx context.Context, agentSpanID uuid.UUID, st
|
||||
span.TenantID = store.MasterTenantID
|
||||
}
|
||||
|
||||
collector.EmitSpan(span)
|
||||
collector.EmitSpan(tracing.RedactSpan(ctx, span))
|
||||
}
|
||||
|
||||
// emitAgentSpanEnd finalizes the running root agent span with results.
|
||||
@@ -433,7 +433,7 @@ func (l *Loop) emitAgentSpanEnd(ctx context.Context, agentSpanID uuid.UUID, star
|
||||
// with child spans that directly report model usage.
|
||||
}
|
||||
|
||||
collector.EmitSpanUpdate(agentSpanID, traceID, updates)
|
||||
collector.EmitSpanUpdate(agentSpanID, traceID, tracing.RedactSpanUpdates(ctx, updates))
|
||||
}
|
||||
|
||||
// previewLimitForVerbose returns the preview character limit based on verbose mode.
|
||||
|
||||
@@ -666,11 +666,13 @@ type RunRequest struct {
|
||||
OnTraceCreated func(traceID uuid.UUID)
|
||||
|
||||
// Delegation context (set when running as a delegate agent)
|
||||
DelegationID string // delegation ID for event correlation
|
||||
TeamID string // team ID (if delegation is team-scoped)
|
||||
TeamTaskID string // team task ID (if delegation has an associated task)
|
||||
ParentAgentID string // parent agent key that initiated the delegation
|
||||
LeaderAgentID string // leader agent UUID for member memory read fallback
|
||||
DelegationID string // delegation ID for event correlation
|
||||
DelegateInputsPath string // runtime-only read-only staged inputs root
|
||||
DelegateOutputsPath string // runtime-only writable exchange workspace
|
||||
TeamID string // team ID (if delegation is team-scoped)
|
||||
TeamTaskID string // team task ID (if delegation has an associated task)
|
||||
ParentAgentID string // parent agent key that initiated the delegation
|
||||
LeaderAgentID string // leader agent UUID for member memory read fallback
|
||||
|
||||
// Workspace scope propagation (set by delegation, read by workspace tools)
|
||||
WorkspaceChannel string
|
||||
@@ -678,6 +680,12 @@ type RunRequest struct {
|
||||
// TeamWorkspace overrides the member agent's workspace with the team's workspace
|
||||
// so file operations (read/write/image/audio) use the shared team directory.
|
||||
TeamWorkspace string
|
||||
|
||||
// enrichedInputMessage is populated by the media stage and consumed by the
|
||||
// first persistence checkpoint. It keeps current-turn MediaRefs and logical
|
||||
// tags durable without storing inline image bytes.
|
||||
enrichedInputMessage providers.Message
|
||||
hasEnrichedInputMessage bool
|
||||
}
|
||||
|
||||
// RunResult is the output of a completed agent run.
|
||||
|
||||
+162
-47
@@ -8,6 +8,7 @@ import (
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
@@ -299,13 +300,23 @@ func copyMediaFile(src, dst string) error {
|
||||
return out.Close()
|
||||
}
|
||||
|
||||
// enrichDocumentPaths updates the last user message to include persisted file paths
|
||||
// in <media:document> tags. This allows skills (e.g. pdf skill via exec) to access
|
||||
// the file directly, matching how Claude Code skills work with file paths.
|
||||
func (l *Loop) enrichDocumentPaths(messages []providers.Message, refs []providers.MediaRef) {
|
||||
// enrichDocumentPaths updates document tags with exact media IDs and
|
||||
// workspace-relative paths. It upgrades historical tags and pairs current refs
|
||||
// with the last user message. Paths outside the active workspace are omitted.
|
||||
func (l *Loop) enrichDocumentPaths(messages []providers.Message, refs []providers.MediaRef, workspace string) {
|
||||
if len(messages) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// Upgrade historical tags as they re-enter the prompt. Current-turn refs
|
||||
// are handled below because they are not yet attached to message history.
|
||||
for i := range messages {
|
||||
if messages[i].Role != "user" || len(messages[i].MediaRefs) == 0 {
|
||||
continue
|
||||
}
|
||||
messages[i].Content = l.enrichDocumentTagContent(messages[i].Content, messages[i].MediaRefs, workspace)
|
||||
}
|
||||
|
||||
lastIdx := -1
|
||||
for i := len(messages) - 1; i >= 0; i-- {
|
||||
if messages[i].Role == "user" {
|
||||
@@ -316,41 +327,62 @@ func (l *Loop) enrichDocumentPaths(messages []providers.Message, refs []provider
|
||||
if lastIdx < 0 {
|
||||
return
|
||||
}
|
||||
messages[lastIdx].Content = l.enrichDocumentTagContent(messages[lastIdx].Content, refs, workspace)
|
||||
}
|
||||
|
||||
content := messages[lastIdx].Content
|
||||
func (l *Loop) enrichDocumentTagContent(content string, refs []providers.MediaRef, workspace string) string {
|
||||
for _, ref := range refs {
|
||||
if ref.Kind != "document" {
|
||||
continue
|
||||
}
|
||||
p := ref.Path
|
||||
if p == "" && l.mediaStore != nil {
|
||||
var err error
|
||||
p, err = l.mediaStore.LoadPath(ref.ID)
|
||||
if err != nil {
|
||||
continue
|
||||
if loaded, err := l.mediaStore.LoadPath(ref.ID); err == nil {
|
||||
p = loaded
|
||||
}
|
||||
}
|
||||
if p == "" {
|
||||
logical := logicalWorkspaceMediaPath(workspace, p)
|
||||
|
||||
updateTag := func(tag string) string {
|
||||
tag = setTagAttr(tag, "id", ref.ID)
|
||||
if logical == "" {
|
||||
return removeTagAttr(tag, "path")
|
||||
}
|
||||
return setTagAttr(tag, "path", logical)
|
||||
}
|
||||
|
||||
// Prefer a tag already carrying the exact media ID. This also upgrades
|
||||
// legacy absolute paths without relying on attribute order.
|
||||
var replaced bool
|
||||
content, replaced = replaceFirstMediaTag(content, "<media:document", func(tag string) bool {
|
||||
return tagHasAttrValue(tag, "id", ref.ID)
|
||||
}, updateTag)
|
||||
if replaced {
|
||||
continue
|
||||
}
|
||||
pathAttr := fmt.Sprintf(" path=%q", p)
|
||||
|
||||
// Match first <media:document> without a path — covers bare, named, and file= variants.
|
||||
// Fallback: pair the next tag without an ID with this persisted ref.
|
||||
content, _ = replaceFirstMediaTag(content, "<media:document", func(tag string) bool {
|
||||
return !tagHasAttr(tag, "path")
|
||||
}, func(tag string) string {
|
||||
return appendTagAttrs(tag, pathAttr)
|
||||
})
|
||||
return !tagHasAttr(tag, "id")
|
||||
}, updateTag)
|
||||
}
|
||||
messages[lastIdx].Content = content
|
||||
return content
|
||||
}
|
||||
|
||||
// enrichAudioIDs updates the last user message to embed persisted media IDs
|
||||
// in <media:audio> and <media:voice> tags so the LLM can reference them.
|
||||
func (l *Loop) enrichAudioIDs(messages []providers.Message, refs []providers.MediaRef) {
|
||||
// enrichAudioIDs updates audio/voice tags with exact media IDs and logical
|
||||
// workspace paths. Historical tags are upgraded when they re-enter the prompt.
|
||||
func (l *Loop) enrichAudioIDs(messages []providers.Message, refs []providers.MediaRef, workspace string) {
|
||||
if len(messages) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
for i := range messages {
|
||||
if messages[i].Role != "user" || len(messages[i].MediaRefs) == 0 {
|
||||
continue
|
||||
}
|
||||
messages[i].Content = l.enrichAudioTagContent(messages[i].Content, messages[i].MediaRefs, workspace)
|
||||
}
|
||||
|
||||
lastIdx := -1
|
||||
for i := len(messages) - 1; i >= 0; i-- {
|
||||
if messages[i].Role == "user" {
|
||||
@@ -361,31 +393,55 @@ func (l *Loop) enrichAudioIDs(messages []providers.Message, refs []providers.Med
|
||||
if lastIdx < 0 {
|
||||
return
|
||||
}
|
||||
messages[lastIdx].Content = l.enrichAudioTagContent(messages[lastIdx].Content, refs, workspace)
|
||||
}
|
||||
|
||||
content := messages[lastIdx].Content
|
||||
func (l *Loop) enrichAudioTagContent(content string, refs []providers.MediaRef, workspace string) string {
|
||||
for _, ref := range refs {
|
||||
if ref.Kind != "audio" {
|
||||
continue
|
||||
}
|
||||
idAttr := fmt.Sprintf(" id=%q", ref.ID)
|
||||
p := ref.Path
|
||||
if p == "" && l.mediaStore != nil {
|
||||
if loaded, err := l.mediaStore.LoadPath(ref.ID); err == nil {
|
||||
p = loaded
|
||||
}
|
||||
}
|
||||
logical := logicalWorkspaceMediaPath(workspace, p)
|
||||
updateTag := func(tag string) string {
|
||||
tag = setTagAttr(tag, "id", ref.ID)
|
||||
if logical == "" {
|
||||
return removeTagAttr(tag, "path")
|
||||
}
|
||||
return setTagAttr(tag, "path", logical)
|
||||
}
|
||||
|
||||
// Upgrade a historical tag already carrying this exact ID first.
|
||||
var replaced bool
|
||||
content, replaced = replaceFirstMediaTag(content, "<media:audio", func(tag string) bool {
|
||||
return !tagHasAttr(tag, "id")
|
||||
}, func(tag string) string {
|
||||
return appendTagAttrs(tag, idAttr)
|
||||
})
|
||||
return tagHasAttrValue(tag, "id", ref.ID)
|
||||
}, updateTag)
|
||||
if replaced {
|
||||
continue
|
||||
}
|
||||
content, replaced = replaceFirstMediaTag(content, "<media:voice", func(tag string) bool {
|
||||
return tagHasAttrValue(tag, "id", ref.ID)
|
||||
}, updateTag)
|
||||
if replaced {
|
||||
continue
|
||||
}
|
||||
|
||||
content, _ = replaceFirstMediaTag(content, "<media:voice", func(tag string) bool {
|
||||
// Pair a current ref with the next unowned audio/voice tag.
|
||||
content, replaced = replaceFirstMediaTag(content, "<media:audio", func(tag string) bool {
|
||||
return !tagHasAttr(tag, "id")
|
||||
}, func(tag string) string {
|
||||
return appendTagAttrs(tag, idAttr)
|
||||
})
|
||||
}, updateTag)
|
||||
if !replaced {
|
||||
content, _ = replaceFirstMediaTag(content, "<media:voice", func(tag string) bool {
|
||||
return !tagHasAttr(tag, "id")
|
||||
}, updateTag)
|
||||
}
|
||||
}
|
||||
messages[lastIdx].Content = content
|
||||
return content
|
||||
}
|
||||
|
||||
// enrichVideoIDs updates the last user message to embed persisted media IDs
|
||||
@@ -422,11 +478,11 @@ func (l *Loop) enrichVideoIDs(messages []providers.Message, refs []providers.Med
|
||||
}
|
||||
|
||||
// enrichImageIDs updates the last user message to embed persisted media IDs
|
||||
// and file paths in <media:image> tags so the LLM knows images were received
|
||||
// and stored. The path attribute allows tools called via MCP bridge (e.g.
|
||||
// and logical workspace paths in <media:image> tags so the LLM knows images
|
||||
// were received and stored. The path attribute allows tools called via MCP bridge (e.g.
|
||||
// claude-cli) to access images via read_image(path=...) even though the
|
||||
// bridge context does not carry WithMediaImages.
|
||||
func (l *Loop) enrichImageIDs(messages []providers.Message, refs []providers.MediaRef) {
|
||||
func (l *Loop) enrichImageIDs(messages []providers.Message, refs []providers.MediaRef, workspace string) {
|
||||
if len(messages) == 0 {
|
||||
return
|
||||
}
|
||||
@@ -448,8 +504,8 @@ func (l *Loop) enrichImageIDs(messages []providers.Message, refs []providers.Med
|
||||
}
|
||||
idAttr := fmt.Sprintf(" id=%q", ref.ID)
|
||||
pathAttr := ""
|
||||
if ref.Path != "" {
|
||||
pathAttr = fmt.Sprintf(" path=%q", ref.Path)
|
||||
if logical := logicalWorkspaceMediaPath(workspace, ref.Path); logical != "" {
|
||||
pathAttr = fmt.Sprintf(" path=%q", logical)
|
||||
}
|
||||
|
||||
content, _ = replaceFirstMediaTag(content, "<media:image", func(tag string) bool {
|
||||
@@ -465,15 +521,12 @@ func (l *Loop) enrichImageIDs(messages []providers.Message, refs []providers.Med
|
||||
messages[lastIdx].Content = content
|
||||
}
|
||||
|
||||
// enrichImagePaths updates ALL user messages to include persisted file paths
|
||||
// enrichImagePaths updates ALL user messages to include logical workspace paths
|
||||
// in <media:image> tags. This enables the LLM to call read_image(path=...)
|
||||
// to analyze images without inline base64 (saving context tokens).
|
||||
// Unlike enrichImageIDs (last user message only), this enriches ALL messages
|
||||
// so historical images from prior turns are also accessible via file path.
|
||||
func (l *Loop) enrichImagePaths(messages []providers.Message) {
|
||||
if l.mediaStore == nil {
|
||||
return
|
||||
}
|
||||
func (l *Loop) enrichImagePaths(messages []providers.Message, workspace string) {
|
||||
for i := range messages {
|
||||
if messages[i].Role != "user" || len(messages[i].MediaRefs) == 0 {
|
||||
continue
|
||||
@@ -485,7 +538,7 @@ func (l *Loop) enrichImagePaths(messages []providers.Message) {
|
||||
continue
|
||||
}
|
||||
p := ref.Path
|
||||
if p == "" {
|
||||
if p == "" && l.mediaStore != nil {
|
||||
var err error
|
||||
p, err = l.mediaStore.LoadPath(ref.ID)
|
||||
if err != nil {
|
||||
@@ -495,14 +548,26 @@ func (l *Loop) enrichImagePaths(messages []providers.Message) {
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
pathAttr := fmt.Sprintf(" path=%q", p)
|
||||
logical := logicalWorkspaceMediaPath(workspace, p)
|
||||
if logical == "" {
|
||||
var stripped bool
|
||||
content, stripped = replaceFirstMediaTag(content, "<media:image", func(tag string) bool {
|
||||
return tagHasAttrValue(tag, "id", ref.ID) && tagHasAttr(tag, "path")
|
||||
}, func(tag string) string {
|
||||
return removeTagAttr(tag, "path")
|
||||
})
|
||||
changed = changed || stripped
|
||||
continue
|
||||
}
|
||||
|
||||
// Prefer tags that already carry the matching media ID.
|
||||
// Prefer tags that already carry the matching media ID. Replacing an
|
||||
// existing path also upgrades legacy messages that persisted absolute
|
||||
// host paths before logical media paths became the prompt contract.
|
||||
var replaced bool
|
||||
content, replaced = replaceFirstMediaTag(content, "<media:image", func(tag string) bool {
|
||||
return tagHasAttrValue(tag, "id", ref.ID) && !tagHasAttr(tag, "path")
|
||||
return tagHasAttrValue(tag, "id", ref.ID)
|
||||
}, func(tag string) string {
|
||||
return appendTagAttrs(tag, pathAttr)
|
||||
return setTagAttr(tag, "path", logical)
|
||||
})
|
||||
if replaced {
|
||||
changed = true
|
||||
@@ -513,7 +578,7 @@ func (l *Loop) enrichImagePaths(messages []providers.Message) {
|
||||
content, replaced = replaceFirstMediaTag(content, "<media:image", func(tag string) bool {
|
||||
return !tagHasAttr(tag, "id")
|
||||
}, func(tag string) string {
|
||||
return appendTagAttrs(tag, fmt.Sprintf(` id=%q`, ref.ID), pathAttr)
|
||||
return appendTagAttrs(tag, fmt.Sprintf(` id=%q`, ref.ID), fmt.Sprintf(` path=%q`, logical))
|
||||
})
|
||||
if replaced {
|
||||
changed = true
|
||||
@@ -525,6 +590,26 @@ func (l *Loop) enrichImagePaths(messages []providers.Message) {
|
||||
}
|
||||
}
|
||||
|
||||
// logicalWorkspaceMediaPath converts a persisted host path into the stable path
|
||||
// contract exposed to models. Paths outside the active workspace are omitted.
|
||||
func logicalWorkspaceMediaPath(workspace, mediaPath string) string {
|
||||
if workspace == "" || mediaPath == "" {
|
||||
return ""
|
||||
}
|
||||
if !filepath.IsAbs(mediaPath) {
|
||||
clean := filepath.Clean(mediaPath)
|
||||
if clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) {
|
||||
return ""
|
||||
}
|
||||
return filepath.ToSlash(clean)
|
||||
}
|
||||
rel, err := filepath.Rel(filepath.Clean(workspace), filepath.Clean(mediaPath))
|
||||
if err != nil || rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
||||
return ""
|
||||
}
|
||||
return filepath.ToSlash(rel)
|
||||
}
|
||||
|
||||
// mediaKindFromMime returns the media kind ("image", "video", "audio", "document")
|
||||
// based on MIME type prefix.
|
||||
func mediaKindFromMime(mime string) string {
|
||||
@@ -580,6 +665,36 @@ func appendTagAttrs(tag string, attrs ...string) string {
|
||||
return strings.TrimSuffix(tag, ">") + strings.Join(attrs, "") + ">"
|
||||
}
|
||||
|
||||
func setTagAttr(tag, attr, value string) string {
|
||||
prefix := " " + attr + `="`
|
||||
start := strings.Index(tag, prefix)
|
||||
if start < 0 {
|
||||
return appendTagAttrs(tag, fmt.Sprintf(` %s=%q`, attr, value))
|
||||
}
|
||||
valueStart := start + len(prefix)
|
||||
valueEnd := strings.IndexByte(tag[valueStart:], '"')
|
||||
if valueEnd < 0 {
|
||||
return tag
|
||||
}
|
||||
valueEnd += valueStart
|
||||
quoted := strconv.Quote(value)
|
||||
return tag[:valueStart] + quoted[1:len(quoted)-1] + tag[valueEnd:]
|
||||
}
|
||||
|
||||
func removeTagAttr(tag, attr string) string {
|
||||
prefix := " " + attr + `="`
|
||||
start := strings.Index(tag, prefix)
|
||||
if start < 0 {
|
||||
return tag
|
||||
}
|
||||
valueStart := start + len(prefix)
|
||||
valueEnd := strings.IndexByte(tag[valueStart:], '"')
|
||||
if valueEnd < 0 {
|
||||
return tag
|
||||
}
|
||||
return tag[:start] + tag[valueStart+valueEnd+1:]
|
||||
}
|
||||
|
||||
// maxMediaReloadMessages is the default number of recent messages with image MediaRefs
|
||||
// to reload for LLM vision context.
|
||||
const maxMediaReloadMessages = 5
|
||||
|
||||
+216
-31
@@ -1,13 +1,16 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/bus"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/media"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/providers"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/tools"
|
||||
)
|
||||
|
||||
// TestEnrichImageIDs_BareTag verifies enrichment of a bare <media:image> tag
|
||||
@@ -20,15 +23,62 @@ func TestEnrichImageIDs_BareTag(t *testing.T) {
|
||||
refs := []providers.MediaRef{{ID: "img-1", Kind: "image", Path: "/tmp/a.jpg"}}
|
||||
|
||||
var loop Loop
|
||||
loop.enrichImageIDs(messages, refs)
|
||||
loop.enrichImageIDs(messages, refs, "/tmp")
|
||||
|
||||
got := messages[0].Content
|
||||
want := `check <media:image id="img-1" path="/tmp/a.jpg">`
|
||||
want := `check <media:image id="img-1" path="a.jpg">`
|
||||
if got != want {
|
||||
t.Fatalf("bare tag enrichment:\n got %q\nwant %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnrichInputMedia_DelegateRehomesCallerAttachment(t *testing.T) {
|
||||
callerWorkspace := t.TempDir()
|
||||
delegateWorkspace := t.TempDir()
|
||||
source := filepath.Join(callerWorkspace, "reference.jpg")
|
||||
if err := os.WriteFile(source, []byte("reference"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
req := &RunRequest{
|
||||
SessionKey: "delegate:parent:cppai-media:test",
|
||||
RunKind: "delegate",
|
||||
Media: []bus.MediaFile{{
|
||||
Path: source,
|
||||
MimeType: "image/jpeg",
|
||||
Filename: "reference.jpg",
|
||||
}},
|
||||
}
|
||||
messages := []providers.Message{{
|
||||
Role: "user",
|
||||
Content: "Use this exact reference: " + source,
|
||||
}}
|
||||
ctx := tools.WithToolWorkspace(context.Background(), delegateWorkspace)
|
||||
|
||||
loop := Loop{id: "cppai-media"}
|
||||
_, gotMessages, refs := loop.enrichInputMedia(ctx, req, messages)
|
||||
|
||||
if len(refs) != 1 {
|
||||
t.Fatalf("refs = %#v, want one delegate-owned attachment", refs)
|
||||
}
|
||||
if !strings.HasPrefix(refs[0].Path, filepath.Join(delegateWorkspace, ".uploads")+string(filepath.Separator)) {
|
||||
t.Fatalf("persisted path = %q, want under delegate workspace %q", refs[0].Path, delegateWorkspace)
|
||||
}
|
||||
if strings.Contains(gotMessages[0].Content, source) {
|
||||
t.Fatalf("delegate message still exposes caller path: %q", gotMessages[0].Content)
|
||||
}
|
||||
logicalPath := logicalWorkspaceMediaPath(delegateWorkspace, refs[0].Path)
|
||||
if logicalPath == "" || !strings.Contains(gotMessages[0].Content, logicalPath) {
|
||||
t.Fatalf("delegate message = %q, want logical copied path %q", gotMessages[0].Content, logicalPath)
|
||||
}
|
||||
if strings.Contains(gotMessages[0].Content, delegateWorkspace) {
|
||||
t.Fatalf("delegate message leaked delegate workspace: %q", gotMessages[0].Content)
|
||||
}
|
||||
if !strings.Contains(gotMessages[0].Content, `<media:image`) {
|
||||
t.Fatalf("delegate message = %q, want explicit image attachment tag", gotMessages[0].Content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnrichImageIDs_PreservesExistingTagAttributes(t *testing.T) {
|
||||
messages := []providers.Message{{
|
||||
Role: "user",
|
||||
@@ -41,7 +91,7 @@ func TestEnrichImageIDs_PreservesExistingTagAttributes(t *testing.T) {
|
||||
}}
|
||||
|
||||
var loop Loop
|
||||
loop.enrichImageIDs(messages, refs)
|
||||
loop.enrichImageIDs(messages, refs, "/tmp")
|
||||
|
||||
got := messages[0].Content
|
||||
if !strings.Contains(got, `url="https://cdn.discordapp.com/attachments/1/2/photo.jpg"`) {
|
||||
@@ -50,7 +100,7 @@ func TestEnrichImageIDs_PreservesExistingTagAttributes(t *testing.T) {
|
||||
if !strings.Contains(got, `id="image-1"`) {
|
||||
t.Fatalf("expected id attribute to be added, got %q", got)
|
||||
}
|
||||
if !strings.Contains(got, `path="/tmp/photo.jpg"`) {
|
||||
if !strings.Contains(got, `path="photo.jpg"`) {
|
||||
t.Fatalf("expected path attribute to be added, got %q", got)
|
||||
}
|
||||
}
|
||||
@@ -66,7 +116,7 @@ func TestEnrichImageIDs_SkipsAlreadyEnriched(t *testing.T) {
|
||||
refs := []providers.MediaRef{{ID: "new-id", Kind: "image", Path: "/new/path.jpg"}}
|
||||
|
||||
var loop Loop
|
||||
loop.enrichImageIDs(messages, refs)
|
||||
loop.enrichImageIDs(messages, refs, "/new")
|
||||
|
||||
if messages[0].Content != original {
|
||||
t.Fatalf("already-enriched tag should not be modified:\n got %q\nwant %q", messages[0].Content, original)
|
||||
@@ -111,9 +161,9 @@ func testMediaStore(t *testing.T) *media.Store {
|
||||
return s
|
||||
}
|
||||
|
||||
// TestEnrichImagePaths_NoDoubleEnrich verifies that historical messages with
|
||||
// url+id+path are not re-enriched on subsequent turns.
|
||||
func TestEnrichImagePaths_NoDoubleEnrich(t *testing.T) {
|
||||
// TestEnrichImagePaths_UpgradesLegacyAbsolutePath verifies that historical
|
||||
// host paths are replaced by the current logical workspace contract.
|
||||
func TestEnrichImagePaths_UpgradesLegacyAbsolutePath(t *testing.T) {
|
||||
original := `<media:image url="https://cdn.example.com/photo.jpg" id="img-1" path="/workspace/.uploads/img-1.jpg">`
|
||||
messages := []providers.Message{{
|
||||
Role: "user",
|
||||
@@ -126,10 +176,69 @@ func TestEnrichImagePaths_NoDoubleEnrich(t *testing.T) {
|
||||
}}
|
||||
|
||||
loop := Loop{mediaStore: testMediaStore(t)}
|
||||
loop.enrichImagePaths(messages)
|
||||
loop.enrichImagePaths(messages, "/workspace")
|
||||
|
||||
if messages[0].Content != original {
|
||||
t.Fatalf("double-enrichment detected:\n got %q\nwant %q", messages[0].Content, original)
|
||||
want := `<media:image url="https://cdn.example.com/photo.jpg" id="img-1" path=".uploads/img-1.jpg">`
|
||||
if messages[0].Content != want {
|
||||
t.Fatalf("legacy path was not upgraded:\n got %q\nwant %q", messages[0].Content, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnrichImagePathsEscapesLogicalPathAttribute(t *testing.T) {
|
||||
workspace := t.TempDir()
|
||||
mediaPath := filepath.Join(workspace, ".uploads", `photo"quoted.jpg`)
|
||||
messages := []providers.Message{{
|
||||
Role: "user",
|
||||
Content: `<media:image id="img-1" path="/legacy/photo.jpg">`,
|
||||
MediaRefs: []providers.MediaRef{{
|
||||
ID: "img-1",
|
||||
Kind: "image",
|
||||
Path: mediaPath,
|
||||
}},
|
||||
}}
|
||||
|
||||
var loop Loop
|
||||
loop.enrichImagePaths(messages, workspace)
|
||||
|
||||
want := `<media:image id="img-1" path=".uploads/photo\"quoted.jpg">`
|
||||
if messages[0].Content != want {
|
||||
t.Fatalf("escaped path = %q, want %q", messages[0].Content, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogicalWorkspaceMediaPathRejectsOutsideWorkspace(t *testing.T) {
|
||||
workspace := filepath.Join(t.TempDir(), "agent")
|
||||
inside := filepath.Join(workspace, ".uploads", "photo.jpg")
|
||||
outside := filepath.Join(filepath.Dir(workspace), "other-agent", ".uploads", "photo.jpg")
|
||||
|
||||
if got := logicalWorkspaceMediaPath(workspace, inside); got != ".uploads/photo.jpg" {
|
||||
t.Fatalf("inside path = %q, want %q", got, ".uploads/photo.jpg")
|
||||
}
|
||||
if got := logicalWorkspaceMediaPath(workspace, outside); got != "" {
|
||||
t.Fatalf("outside path = %q, want omitted", got)
|
||||
}
|
||||
if got := logicalWorkspaceMediaPath(workspace, "../escape.jpg"); got != "" {
|
||||
t.Fatalf("traversal path = %q, want omitted", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnrichImagePathsStripsOutsideWorkspacePath(t *testing.T) {
|
||||
messages := []providers.Message{{
|
||||
Role: "user",
|
||||
Content: `<media:image id="img-1" path="/workspace/other-agent/.uploads/photo.jpg">`,
|
||||
MediaRefs: []providers.MediaRef{{
|
||||
ID: "img-1",
|
||||
Kind: "image",
|
||||
Path: "/workspace/other-agent/.uploads/photo.jpg",
|
||||
}},
|
||||
}}
|
||||
|
||||
var loop Loop
|
||||
loop.enrichImagePaths(messages, "/workspace/current-agent")
|
||||
|
||||
want := `<media:image id="img-1">`
|
||||
if messages[0].Content != want {
|
||||
t.Fatalf("outside workspace path was not stripped: got %q, want %q", messages[0].Content, want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -148,10 +257,10 @@ func TestEnrichImagePaths_AttributeOrderIndependence(t *testing.T) {
|
||||
}}
|
||||
|
||||
loop := Loop{mediaStore: testMediaStore(t)}
|
||||
loop.enrichImagePaths(messages)
|
||||
loop.enrichImagePaths(messages, "/workspace")
|
||||
|
||||
got := messages[0].Content
|
||||
if !strings.Contains(got, `path="/workspace/.uploads/img-1.jpg"`) {
|
||||
if !strings.Contains(got, `path=".uploads/img-1.jpg"`) {
|
||||
t.Fatalf("expected path to be added regardless of attribute order, got %q", got)
|
||||
}
|
||||
if !strings.Contains(got, `url="https://cdn.example.com/photo.jpg"`) {
|
||||
@@ -173,9 +282,9 @@ func TestEnrichImageIDs_MultipleRefs(t *testing.T) {
|
||||
}
|
||||
|
||||
var loop Loop
|
||||
loop.enrichImageIDs(messages, refs)
|
||||
loop.enrichImageIDs(messages, refs, "/tmp")
|
||||
|
||||
want := `first <media:image id="img-a" path="/tmp/a.jpg">` + "\n" + `second <media:image id="img-b" path="/tmp/b.jpg">`
|
||||
want := `first <media:image id="img-a" path="a.jpg">` + "\n" + `second <media:image id="img-b" path="b.jpg">`
|
||||
if messages[0].Content != want {
|
||||
t.Fatalf("multi-ref alignment:\n got %q\nwant %q", messages[0].Content, want)
|
||||
}
|
||||
@@ -223,53 +332,129 @@ func TestEnrichImagePaths_MultipleRefsKeepTagAlignment(t *testing.T) {
|
||||
|
||||
var loop Loop
|
||||
loop.mediaStore = mediaStore
|
||||
loop.enrichImagePaths(messages)
|
||||
loop.enrichImagePaths(messages, "/persisted")
|
||||
|
||||
want := `first <media:image id="` + idA + `" path="` + pathA + `">` +
|
||||
"\n" + `second <media:image id="` + idB + `" path="` + pathB + `">`
|
||||
want := `first <media:image id="` + idA + `" path="a.jpg">` +
|
||||
"\n" + `second <media:image id="` + idB + `" path="b.jpg">`
|
||||
if messages[0].Content != want {
|
||||
t.Fatalf("enrichImagePaths() content = %q, want %q", messages[0].Content, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnrichDocumentPaths_MultipleRefs(t *testing.T) {
|
||||
workspace := t.TempDir()
|
||||
messages := []providers.Message{{
|
||||
Role: "user",
|
||||
Content: "first <media:document>\nsecond <media:document>",
|
||||
}}
|
||||
refs := []providers.MediaRef{
|
||||
{ID: "doc-a", Kind: "document", Path: "/tmp/a.pdf"},
|
||||
{ID: "doc-b", Kind: "document", Path: "/tmp/b.pdf"},
|
||||
{ID: "doc-a", Kind: "document", Path: filepath.Join(workspace, ".uploads", "a.pdf")},
|
||||
{ID: "doc-b", Kind: "document", Path: filepath.Join(workspace, ".uploads", "b.pdf")},
|
||||
}
|
||||
|
||||
var loop Loop
|
||||
loop.enrichDocumentPaths(messages, refs)
|
||||
loop.enrichDocumentPaths(messages, refs, workspace)
|
||||
|
||||
want := `first <media:document path="/tmp/a.pdf">` + "\n" + `second <media:document path="/tmp/b.pdf">`
|
||||
want := `first <media:document id="doc-a" path=".uploads/a.pdf">` + "\n" +
|
||||
`second <media:document id="doc-b" path=".uploads/b.pdf">`
|
||||
if messages[0].Content != want {
|
||||
t.Fatalf("multi-ref alignment:\n got %q\nwant %q", messages[0].Content, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnrichDocumentPathsStripsOutsideWorkspacePath(t *testing.T) {
|
||||
workspace := filepath.Join(t.TempDir(), "current-agent")
|
||||
outside := filepath.Join(filepath.Dir(workspace), "other-agent", ".uploads", "secret.pdf")
|
||||
messages := []providers.Message{{
|
||||
Role: "user",
|
||||
Content: `<media:document name="secret.pdf" path="/legacy/secret.pdf">`,
|
||||
}}
|
||||
refs := []providers.MediaRef{{
|
||||
ID: "doc-1",
|
||||
Kind: "document",
|
||||
Path: outside,
|
||||
}}
|
||||
|
||||
var loop Loop
|
||||
loop.enrichDocumentPaths(messages, refs, workspace)
|
||||
|
||||
want := `<media:document name="secret.pdf" id="doc-1">`
|
||||
if messages[0].Content != want {
|
||||
t.Fatalf("outside workspace path was not stripped: got %q, want %q", messages[0].Content, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnrichDocumentPathsUpgradesHistoricalAbsolutePath(t *testing.T) {
|
||||
workspace := t.TempDir()
|
||||
docPath := filepath.Join(workspace, ".uploads", "history.pdf")
|
||||
messages := []providers.Message{
|
||||
{
|
||||
Role: "user",
|
||||
Content: `<media:document id="doc-old" path="/legacy/history.pdf">`,
|
||||
MediaRefs: []providers.MediaRef{{
|
||||
ID: "doc-old",
|
||||
Kind: "document",
|
||||
Path: docPath,
|
||||
}},
|
||||
},
|
||||
{Role: "user", Content: "follow up"},
|
||||
}
|
||||
|
||||
var loop Loop
|
||||
loop.enrichDocumentPaths(messages, nil, workspace)
|
||||
|
||||
want := `<media:document id="doc-old" path=".uploads/history.pdf">`
|
||||
if messages[0].Content != want {
|
||||
t.Fatalf("historical path was not upgraded: got %q, want %q", messages[0].Content, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnrichAudioIDs_MultipleRefs(t *testing.T) {
|
||||
workspace := t.TempDir()
|
||||
messages := []providers.Message{{
|
||||
Role: "user",
|
||||
Content: "first <media:audio>\nsecond <media:audio>",
|
||||
}}
|
||||
refs := []providers.MediaRef{
|
||||
{ID: "aud-a", Kind: "audio"},
|
||||
{ID: "aud-b", Kind: "audio"},
|
||||
{ID: "aud-a", Kind: "audio", Path: filepath.Join(workspace, ".uploads", "a.mp3")},
|
||||
{ID: "aud-b", Kind: "audio", Path: filepath.Join(workspace, ".uploads", "b.mp3")},
|
||||
}
|
||||
|
||||
var loop Loop
|
||||
loop.enrichAudioIDs(messages, refs)
|
||||
loop.enrichAudioIDs(messages, refs, workspace)
|
||||
|
||||
want := `first <media:audio id="aud-a">` + "\n" + `second <media:audio id="aud-b">`
|
||||
want := `first <media:audio id="aud-a" path=".uploads/a.mp3">` + "\n" +
|
||||
`second <media:audio id="aud-b" path=".uploads/b.mp3">`
|
||||
if messages[0].Content != want {
|
||||
t.Fatalf("multi-ref alignment:\n got %q\nwant %q", messages[0].Content, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnrichAudioIDsUpgradesHistoricalLogicalPath(t *testing.T) {
|
||||
workspace := t.TempDir()
|
||||
audioPath := filepath.Join(workspace, ".uploads", "history.mp3")
|
||||
messages := []providers.Message{
|
||||
{
|
||||
Role: "user",
|
||||
Content: `<media:voice id="aud-old" path="/legacy/history.mp3">`,
|
||||
MediaRefs: []providers.MediaRef{{
|
||||
ID: "aud-old",
|
||||
Kind: "audio",
|
||||
Path: audioPath,
|
||||
}},
|
||||
},
|
||||
{Role: "user", Content: "follow up"},
|
||||
}
|
||||
|
||||
var loop Loop
|
||||
loop.enrichAudioIDs(messages, nil, workspace)
|
||||
|
||||
want := `<media:voice id="aud-old" path=".uploads/history.mp3">`
|
||||
if messages[0].Content != want {
|
||||
t.Fatalf("historical audio path was not upgraded: got %q, want %q", messages[0].Content, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnrichVideoIDs_MultipleRefs(t *testing.T) {
|
||||
messages := []providers.Message{{
|
||||
Role: "user",
|
||||
@@ -301,9 +486,9 @@ func TestEnrichImageIDs_MoreRefsThanTags(t *testing.T) {
|
||||
}
|
||||
|
||||
var loop Loop
|
||||
loop.enrichImageIDs(messages, refs)
|
||||
loop.enrichImageIDs(messages, refs, "/tmp")
|
||||
|
||||
want := `only one <media:image id="img-a" path="/tmp/a.jpg">`
|
||||
want := `only one <media:image id="img-a" path="a.jpg">`
|
||||
if messages[0].Content != want {
|
||||
t.Fatalf("more refs than tags:\n got %q\nwant %q", messages[0].Content, want)
|
||||
}
|
||||
@@ -320,9 +505,9 @@ func TestEnrichImageIDs_MoreTagsThanRefs(t *testing.T) {
|
||||
}
|
||||
|
||||
var loop Loop
|
||||
loop.enrichImageIDs(messages, refs)
|
||||
loop.enrichImageIDs(messages, refs, "/tmp")
|
||||
|
||||
want := `first <media:image id="img-a" path="/tmp/a.jpg">` + "\n" + `second <media:image>`
|
||||
want := `first <media:image id="img-a" path="a.jpg">` + "\n" + `second <media:image>`
|
||||
if messages[0].Content != want {
|
||||
t.Fatalf("more tags than refs:\n got %q\nwant %q", messages[0].Content, want)
|
||||
}
|
||||
@@ -341,7 +526,7 @@ func TestEnrichAudioIDs_MixedAudioAndVoice(t *testing.T) {
|
||||
}
|
||||
|
||||
var loop Loop
|
||||
loop.enrichAudioIDs(messages, refs)
|
||||
loop.enrichAudioIDs(messages, refs, "")
|
||||
|
||||
want := `hear this <media:audio id="aud-1">` + "\n" + `and this <media:voice id="aud-2">`
|
||||
if messages[0].Content != want {
|
||||
|
||||
@@ -111,4 +111,10 @@ func TestBridgePromptBuilder_OrchestrationTargetContent(t *testing.T) {
|
||||
if !strings.Contains(output, "Does work") {
|
||||
t.Error("output should contain description")
|
||||
}
|
||||
if !strings.Contains(output, "Return generated files or media artifacts to the caller") {
|
||||
t.Error("output should tell delegates to return media artifacts through the caller")
|
||||
}
|
||||
if !strings.Contains(output, "Do not ask the delegatee to send directly to your current chat") {
|
||||
t.Error("output should prevent direct delivery attempts from the delegate channel")
|
||||
}
|
||||
}
|
||||
@@ -98,8 +98,6 @@ type WorkspaceSectionData struct {
|
||||
ActivePath string
|
||||
Scope string
|
||||
Enforced bool
|
||||
ReadOnlyPaths []string
|
||||
SharedPath *string
|
||||
ContextFiles []string
|
||||
EnforcementMsg string
|
||||
}
|
||||
|
||||
@@ -207,7 +207,7 @@ var coreToolSummaries = map[string]string{
|
||||
"session_status": "Show session status (model, tokens, compaction count)",
|
||||
"sessions_history": "Fetch message history for a session",
|
||||
"sessions_send": "Send a message into another session",
|
||||
"read_image": "Analyze images — call with path from <media:image> tags, or a direct HTTP/HTTPS URL via the 'url' parameter",
|
||||
"read_image": "Analyze images — use the exact logical path from <media:image> tags; media_id and direct HTTP/HTTPS URLs are also supported",
|
||||
"read_audio": "Analyze audio — call with media_id from <media:audio> tags",
|
||||
"read_video": "Analyze video — call with media_id from <media:video> tags, or a direct HTTP/HTTPS URL via the 'url' parameter",
|
||||
"create_video": "Generate videos from text descriptions using AI",
|
||||
@@ -673,7 +673,7 @@ func buildToolingSection(toolNames []string, hasSandbox bool, shellDenyGroups ma
|
||||
lines = append(lines,
|
||||
"",
|
||||
"### Media Files",
|
||||
`When users send media (<media:image path="...">, <media:video id="...">, <media:audio id="...">, <media:document path="...">), use the corresponding read_* tool with the path/media_id. For archives (.zip, .tar.gz, etc.), use exec with the document path to inspect/extract the archive.`,
|
||||
`When users send media (<media:image id="..." path="...">, <media:video id="...">, <media:audio id="...">, <media:document path="...">), use the corresponding read_* tool with the exact path/media_id from the tag. Never reconstruct or guess an absolute workspace path. For archives (.zip, .tar.gz, etc.), use exec with the document path to inspect/extract the archive.`,
|
||||
"You have full vision/audio/video capabilities. NEVER say you cannot see images or files.",
|
||||
)
|
||||
}
|
||||
|
||||
@@ -771,6 +771,8 @@ func buildOrchestrationSection(data OrchestrationSectionData) []string {
|
||||
lines = append(lines,
|
||||
"",
|
||||
"Use `delegate` with the agent_key of the target agent. Do NOT invent agent keys.",
|
||||
"Return generated files or media artifacts to the caller through the delegate result.",
|
||||
"Do not ask the delegatee to send directly to your current chat; delegated runs use an internal `delegate` channel and the caller handles final delivery.",
|
||||
"",
|
||||
)
|
||||
return lines
|
||||
|
||||
@@ -18,6 +18,10 @@ const maxSubagentsInList = 30
|
||||
// subagentStatusIcon returns an icon for each subagent task status.
|
||||
func subagentStatusIcon(status string) string {
|
||||
switch status {
|
||||
case "queued":
|
||||
return "⏳"
|
||||
case "waiting_child":
|
||||
return "↪️"
|
||||
case "completed":
|
||||
return "✅"
|
||||
case "failed":
|
||||
@@ -52,18 +56,20 @@ func (c *Channel) handleSubagentsList(ctx context.Context, chatID int64, isGroup
|
||||
return
|
||||
}
|
||||
|
||||
agentKey := c.AgentID()
|
||||
if agentKey == "" {
|
||||
send("Subagent tasks are not available (no agent configured).")
|
||||
rootAgentID, err := c.resolveAgentUUID(ctx)
|
||||
if err != nil {
|
||||
slog.Warn("subagents command: resolve agent UUID failed", "error", err)
|
||||
send("Subagent tasks are not available (agent could not be resolved).")
|
||||
return
|
||||
}
|
||||
|
||||
tasks, err := c.subagentTaskStore.ListByParent(ctx, agentKey, "")
|
||||
tasks, err := c.subagentTaskStore.ListByParent(ctx, rootAgentID, "")
|
||||
if err != nil {
|
||||
slog.Warn("subagents command: ListByParent failed", "error", err)
|
||||
send("Failed to list subagent tasks. Please try again.")
|
||||
return
|
||||
}
|
||||
tasks = filterSelfCloneTasks(tasks)
|
||||
|
||||
if len(tasks) == 0 {
|
||||
send("No subagent tasks found.")
|
||||
@@ -142,7 +148,13 @@ func (c *Channel) handleSubagentDetail(ctx context.Context, chatID int64, text s
|
||||
return
|
||||
}
|
||||
|
||||
task, err := c.subagentTaskStore.Get(ctx, taskID)
|
||||
rootAgentID, err := c.resolveAgentUUID(ctx)
|
||||
if err != nil {
|
||||
slog.Warn("subagent command: resolve agent UUID failed", "error", err)
|
||||
send("Subagent tasks are not available (agent could not be resolved).")
|
||||
return
|
||||
}
|
||||
task, err := c.subagentTaskStore.Get(ctx, rootAgentID, taskID)
|
||||
if err != nil {
|
||||
slog.Warn("subagent command: Get failed", "id", idArg, "error", err)
|
||||
send("Failed to load subagent task. Please try again.")
|
||||
@@ -152,6 +164,10 @@ func (c *Channel) handleSubagentDetail(ctx context.Context, chatID int64, text s
|
||||
send(fmt.Sprintf("Task %q not found. Use /subagents to see available tasks.", idArg[:8]))
|
||||
return
|
||||
}
|
||||
if isDelegationCompletion(task) {
|
||||
send(fmt.Sprintf("Task %q is a delegation result; retrieve it with the delegate tool.", idArg[:8]))
|
||||
return
|
||||
}
|
||||
|
||||
send(formatSubagentDetail(task))
|
||||
}
|
||||
@@ -181,7 +197,13 @@ func (c *Channel) handleSubagentCallback(ctx context.Context, query *telego.Call
|
||||
return
|
||||
}
|
||||
|
||||
task, err := c.subagentTaskStore.Get(ctx, taskID)
|
||||
rootAgentID, err := c.resolveAgentUUID(ctx)
|
||||
if err != nil {
|
||||
slog.Warn("subagent callback: resolve agent UUID failed", "error", err)
|
||||
send("Subagent tasks are not available (agent could not be resolved).")
|
||||
return
|
||||
}
|
||||
task, err := c.subagentTaskStore.Get(ctx, rootAgentID, taskID)
|
||||
if err != nil {
|
||||
slog.Warn("subagent callback: Get failed", "id", taskIDStr, "error", err)
|
||||
send("Failed to load subagent task.")
|
||||
@@ -191,6 +213,10 @@ func (c *Channel) handleSubagentCallback(ctx context.Context, query *telego.Call
|
||||
send(fmt.Sprintf("Task %s not found.", taskIDStr[:8]))
|
||||
return
|
||||
}
|
||||
if isDelegationCompletion(task) {
|
||||
send("This task is a delegation result and is not part of /subagents.")
|
||||
return
|
||||
}
|
||||
|
||||
send(formatSubagentDetail(task))
|
||||
}
|
||||
@@ -218,3 +244,21 @@ func formatSubagentDetail(t *store.SubagentTaskData) string {
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func filterSelfCloneTasks(tasks []store.SubagentTaskData) []store.SubagentTaskData {
|
||||
filtered := tasks[:0]
|
||||
for i := range tasks {
|
||||
if !isDelegationCompletion(&tasks[i]) {
|
||||
filtered = append(filtered, tasks[i])
|
||||
}
|
||||
}
|
||||
return filtered
|
||||
}
|
||||
|
||||
func isDelegationCompletion(task *store.SubagentTaskData) bool {
|
||||
if task == nil || task.Metadata == nil {
|
||||
return false
|
||||
}
|
||||
kind, _ := task.Metadata["completion_kind"].(string)
|
||||
return kind == "delegate"
|
||||
}
|
||||
@@ -0,0 +1,901 @@
|
||||
package childrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrChildRunBusy = errors.New("child_run_busy")
|
||||
ErrChildRunClosed = errors.New("child_run_closed")
|
||||
ErrChildRunDrainTimeout = errors.New("child_run_drain_timeout")
|
||||
ErrChildRunInvalidContinuation = errors.New("child_run_invalid_continuation")
|
||||
ErrChildRunContinuationDepth = errors.New("child_run_continuation_depth")
|
||||
)
|
||||
|
||||
const maxChildRunContinuationDepth = 64
|
||||
|
||||
type ChildRunState string
|
||||
|
||||
const (
|
||||
ChildRunNew ChildRunState = "new"
|
||||
ChildRunQueued ChildRunState = "queued"
|
||||
ChildRunRunning ChildRunState = "running"
|
||||
ChildRunWaitingChild ChildRunState = "waiting_child"
|
||||
ChildRunCompleted ChildRunState = "completed"
|
||||
ChildRunFailed ChildRunState = "failed"
|
||||
ChildRunCancelled ChildRunState = "cancelled"
|
||||
)
|
||||
|
||||
// ChildRunConstraints are rechecked atomically whenever a run or suspended
|
||||
// continuation is granted. Nil root identity means process-limit-only work.
|
||||
//
|
||||
// TaskID, Depth, and MaxDepth are optional for top-level runs, but TaskID and
|
||||
// direct-child lineage are required for synchronous continuation handoff.
|
||||
type ChildRunConstraints struct {
|
||||
TenantID uuid.UUID
|
||||
RootAgentID uuid.UUID
|
||||
RootLimit int
|
||||
TaskID string
|
||||
ParentTaskID string
|
||||
ParentFanout int
|
||||
Depth int
|
||||
MaxDepth int
|
||||
}
|
||||
|
||||
type ChildRunStats struct {
|
||||
Active int
|
||||
Pending int
|
||||
Live int
|
||||
Closed bool
|
||||
}
|
||||
|
||||
type childRunRoot struct {
|
||||
tenant uuid.UUID
|
||||
agent uuid.UUID
|
||||
}
|
||||
|
||||
type childRunParent struct {
|
||||
root childRunRoot
|
||||
parent string
|
||||
}
|
||||
|
||||
// ChildRunAdmission owns process-wide child execution admission. It creates
|
||||
// one dispatcher goroutine and stops it after Close has fully drained.
|
||||
type ChildRunAdmission struct {
|
||||
mu sync.Mutex
|
||||
processLimit int
|
||||
pendingLimit int
|
||||
nextSequence uint64
|
||||
active int
|
||||
live int
|
||||
pendingIndependent int
|
||||
pending []*childRunRequest
|
||||
running map[*ChildRunTicket]*childRunFrame
|
||||
liveTickets map[*ChildRunTicket]struct{}
|
||||
rootActive map[childRunRoot]int
|
||||
parentActive map[childRunParent]int
|
||||
closed bool
|
||||
wake chan struct{}
|
||||
stop chan struct{}
|
||||
dispatcherDone chan struct{}
|
||||
drained chan struct{}
|
||||
drainedOnce sync.Once
|
||||
stopOnce sync.Once
|
||||
}
|
||||
|
||||
// ChildRunTicket represents one independent admission chain. Nested
|
||||
// synchronous continuations reuse this chain and its original FIFO sequence.
|
||||
type ChildRunTicket struct {
|
||||
controller *ChildRunAdmission
|
||||
ctx context.Context
|
||||
sequence uint64
|
||||
maxDepth int
|
||||
state ChildRunState
|
||||
err error
|
||||
|
||||
activated bool
|
||||
cancelRequested bool
|
||||
finished bool
|
||||
live bool
|
||||
started chan struct{}
|
||||
done chan struct{}
|
||||
doneOnce sync.Once
|
||||
stopCancel func() bool
|
||||
|
||||
top *childRunFrame
|
||||
current *childRunFrame
|
||||
stack []*childRunFrame
|
||||
requests map[*childRunRequest]struct{}
|
||||
topRequest *childRunRequest
|
||||
}
|
||||
|
||||
// ChildRunLease is controller-issued execution authority. Continue validates
|
||||
// that the lease is the chain's current owner before it can hand capacity to a
|
||||
// direct synchronous child.
|
||||
type ChildRunLease struct {
|
||||
controller *ChildRunAdmission
|
||||
ticket *ChildRunTicket
|
||||
frame *childRunFrame
|
||||
continueMu sync.Mutex
|
||||
releaseRequested atomic.Bool
|
||||
}
|
||||
|
||||
// ContinuationParent exposes immutable structural lineage for a direct
|
||||
// synchronous continuation. Agent-level spawn policy is enforced by the tool
|
||||
// that owns the child; this lineage only protects admission handoff integrity.
|
||||
func (l *ChildRunLease) ContinuationParent() (taskID string, depth int, ok bool) {
|
||||
if l == nil || l.frame == nil || l.frame.constraints.TaskID == "" {
|
||||
return "", 0, false
|
||||
}
|
||||
return l.frame.constraints.TaskID, l.frame.constraints.Depth, true
|
||||
}
|
||||
|
||||
type childRunFrame struct {
|
||||
constraints ChildRunConstraints
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
run func(context.Context, *ChildRunLease)
|
||||
lease *ChildRunLease
|
||||
state ChildRunState
|
||||
cancelRequested bool
|
||||
stopCancel func() bool
|
||||
}
|
||||
|
||||
type childRunRequest struct {
|
||||
ticket *ChildRunTicket
|
||||
frame *childRunFrame
|
||||
sequence uint64
|
||||
independent bool
|
||||
launch bool
|
||||
activated bool
|
||||
granted chan struct{}
|
||||
cancelled chan struct{}
|
||||
cancelErr error
|
||||
resolved bool
|
||||
}
|
||||
|
||||
func NewChildRunAdmission(processLimit, pendingLimit int) *ChildRunAdmission {
|
||||
if processLimit < 1 {
|
||||
processLimit = 1
|
||||
}
|
||||
if pendingLimit < 0 {
|
||||
pendingLimit = 0
|
||||
}
|
||||
c := &ChildRunAdmission{
|
||||
processLimit: processLimit,
|
||||
pendingLimit: pendingLimit,
|
||||
running: make(map[*ChildRunTicket]*childRunFrame),
|
||||
liveTickets: make(map[*ChildRunTicket]struct{}),
|
||||
rootActive: make(map[childRunRoot]int),
|
||||
parentActive: make(map[childRunParent]int),
|
||||
wake: make(chan struct{}, 1),
|
||||
stop: make(chan struct{}),
|
||||
dispatcherDone: make(chan struct{}),
|
||||
drained: make(chan struct{}),
|
||||
}
|
||||
go c.dispatch()
|
||||
return c
|
||||
}
|
||||
|
||||
// Enqueue reserves one bounded independent pending record but leaves it
|
||||
// dormant. Activate makes the run eligible after its owner installs visible
|
||||
// queued task state.
|
||||
func (c *ChildRunAdmission) Enqueue(
|
||||
ctx context.Context,
|
||||
constraints ChildRunConstraints,
|
||||
run func(context.Context, *ChildRunLease),
|
||||
) (*ChildRunTicket, error) {
|
||||
if run == nil {
|
||||
return nil, fmt.Errorf("child run callback is required")
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.closed {
|
||||
return nil, ErrChildRunClosed
|
||||
}
|
||||
if c.pendingIndependent >= c.pendingLimit {
|
||||
return nil, ErrChildRunBusy
|
||||
}
|
||||
maxDepth := constraints.MaxDepth
|
||||
if maxDepth <= 0 || maxDepth > maxChildRunContinuationDepth {
|
||||
maxDepth = maxChildRunContinuationDepth
|
||||
}
|
||||
if constraints.Depth < 0 || constraints.Depth > maxDepth {
|
||||
return nil, ErrChildRunContinuationDepth
|
||||
}
|
||||
constraints.MaxDepth = maxDepth
|
||||
|
||||
c.nextSequence++
|
||||
ticket := &ChildRunTicket{
|
||||
controller: c,
|
||||
ctx: ctx,
|
||||
sequence: c.nextSequence,
|
||||
maxDepth: maxDepth,
|
||||
state: ChildRunNew,
|
||||
started: make(chan struct{}),
|
||||
done: make(chan struct{}),
|
||||
requests: make(map[*childRunRequest]struct{}),
|
||||
}
|
||||
frame := c.newFrameLocked(ticket, ctx, constraints, run)
|
||||
request := &childRunRequest{
|
||||
ticket: ticket,
|
||||
frame: frame,
|
||||
sequence: ticket.sequence,
|
||||
independent: true,
|
||||
launch: true,
|
||||
granted: make(chan struct{}),
|
||||
cancelled: make(chan struct{}),
|
||||
}
|
||||
ticket.top = frame
|
||||
ticket.stack = []*childRunFrame{frame}
|
||||
ticket.topRequest = request
|
||||
ticket.requests[request] = struct{}{}
|
||||
c.pending = append(c.pending, request)
|
||||
c.pendingIndependent++
|
||||
ticket.stopCancel = context.AfterFunc(ctx, func() {
|
||||
ticket.Cancel()
|
||||
})
|
||||
return ticket, nil
|
||||
}
|
||||
|
||||
func (t *ChildRunTicket) Activate() error {
|
||||
c := t.controller
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
if t.finished || t.cancelRequested {
|
||||
if t.err != nil {
|
||||
return t.err
|
||||
}
|
||||
return context.Canceled
|
||||
}
|
||||
if c.closed {
|
||||
c.cancelTicketLocked(t, ErrChildRunClosed)
|
||||
return ErrChildRunClosed
|
||||
}
|
||||
if t.state != ChildRunNew {
|
||||
return nil
|
||||
}
|
||||
t.activated = true
|
||||
t.state = ChildRunQueued
|
||||
t.top.state = ChildRunQueued
|
||||
t.topRequest.activated = true
|
||||
c.signalLocked()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Cancel decides cancel-versus-grant while holding the controller lock.
|
||||
// Running work is marked cancelled immediately, but its capacity is retained
|
||||
// until the callback actually returns.
|
||||
func (t *ChildRunTicket) Cancel() bool {
|
||||
c := t.controller
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if t.finished || t.cancelRequested {
|
||||
return false
|
||||
}
|
||||
c.cancelTicketLocked(t, context.Canceled)
|
||||
return true
|
||||
}
|
||||
|
||||
func (t *ChildRunTicket) State() ChildRunState {
|
||||
t.controller.mu.Lock()
|
||||
defer t.controller.mu.Unlock()
|
||||
return t.state
|
||||
}
|
||||
|
||||
func (t *ChildRunTicket) Err() error {
|
||||
t.controller.mu.Lock()
|
||||
defer t.controller.mu.Unlock()
|
||||
return t.err
|
||||
}
|
||||
|
||||
func (t *ChildRunTicket) Started() <-chan struct{} { return t.started }
|
||||
func (t *ChildRunTicket) Done() <-chan struct{} { return t.done }
|
||||
|
||||
// Release returns execution capacity exactly once while keeping the ticket's
|
||||
// lifecycle barrier open until the callback returns. Controller-owned wrappers
|
||||
// call it only after execution and immutable result capture, before persistence
|
||||
// and announcement side effects.
|
||||
func (l *ChildRunLease) Release() {
|
||||
if l == nil || l.controller == nil || l.ticket == nil || l.frame == nil {
|
||||
return
|
||||
}
|
||||
c := l.controller
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if !l.releaseRequested.CompareAndSwap(false, true) {
|
||||
return
|
||||
}
|
||||
if c.running[l.ticket] == l.frame && l.ticket.current == l.frame {
|
||||
c.releaseActiveLocked(l.ticket, l.frame)
|
||||
}
|
||||
}
|
||||
|
||||
// Continue synchronously transfers this lease's execution capacity to a direct
|
||||
// child and reacquires the parent's original capacity before returning. The
|
||||
// child executes on the caller's goroutine; no dispatcher execution goroutine
|
||||
// is created for synchronous nesting.
|
||||
func (l *ChildRunLease) Continue(
|
||||
ctx context.Context,
|
||||
constraints ChildRunConstraints,
|
||||
run func(context.Context, *ChildRunLease),
|
||||
) error {
|
||||
if l == nil || l.controller == nil || l.ticket == nil || l.frame == nil {
|
||||
return ErrChildRunInvalidContinuation
|
||||
}
|
||||
if run == nil {
|
||||
return fmt.Errorf("child run callback is required")
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// One executing parent owns one transferable continuation. Concurrent
|
||||
// synchronous calls serialize here; async children use Enqueue instead.
|
||||
l.continueMu.Lock()
|
||||
defer l.continueMu.Unlock()
|
||||
|
||||
c := l.controller
|
||||
c.mu.Lock()
|
||||
constraints, err := c.validateContinuationLocked(l, constraints)
|
||||
if err != nil {
|
||||
c.mu.Unlock()
|
||||
return err
|
||||
}
|
||||
|
||||
parent := l.frame
|
||||
child := c.newFrameLocked(l.ticket, ctx, constraints, run)
|
||||
l.ticket.stack = append(l.ticket.stack, child)
|
||||
parent.state = ChildRunWaitingChild
|
||||
l.ticket.state = ChildRunWaitingChild
|
||||
c.releaseActiveLocked(l.ticket, parent)
|
||||
|
||||
request := c.newContinuationRequestLocked(l.ticket, child)
|
||||
sameRoot := sameRootConstraint(parent.constraints, child.constraints)
|
||||
if sameRoot && c.eligibleLocked(child.constraints) {
|
||||
c.grantLocked(request)
|
||||
} else {
|
||||
c.pending = append(c.pending, request)
|
||||
c.signalLocked()
|
||||
}
|
||||
c.mu.Unlock()
|
||||
|
||||
if err := waitChildRunRequest(request); err != nil {
|
||||
resume := c.finishContinuation(child, err)
|
||||
if resume != nil {
|
||||
if resumeErr := waitChildRunRequest(resume); resumeErr != nil {
|
||||
return resumeErr
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
childErr := invokeChildRun(child)
|
||||
if childErr == nil && child.ctx.Err() != nil {
|
||||
childErr = child.ctx.Err()
|
||||
}
|
||||
resume := c.finishContinuation(child, childErr)
|
||||
if resume != nil {
|
||||
if err := waitChildRunRequest(resume); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return childErr
|
||||
}
|
||||
|
||||
func (c *ChildRunAdmission) Stats() ChildRunStats {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return ChildRunStats{
|
||||
Active: c.active,
|
||||
Pending: len(c.pending),
|
||||
Live: c.live,
|
||||
Closed: c.closed,
|
||||
}
|
||||
}
|
||||
|
||||
// Close is idempotent. The first call closes intake and cancels queued and
|
||||
// active work. A timeout leaves the controller and dispatcher available for a
|
||||
// later drain retry; the dispatcher stops only after every callback returns.
|
||||
func (c *ChildRunAdmission) Close(ctx context.Context) error {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
|
||||
c.mu.Lock()
|
||||
if !c.closed {
|
||||
c.closed = true
|
||||
tickets := make(map[*ChildRunTicket]struct{})
|
||||
for _, request := range c.pending {
|
||||
tickets[request.ticket] = struct{}{}
|
||||
}
|
||||
for ticket := range c.running {
|
||||
tickets[ticket] = struct{}{}
|
||||
}
|
||||
for ticket := range c.liveTickets {
|
||||
tickets[ticket] = struct{}{}
|
||||
}
|
||||
for ticket := range tickets {
|
||||
c.cancelTicketLocked(ticket, ErrChildRunClosed)
|
||||
}
|
||||
c.maybeDrainedLocked()
|
||||
c.signalLocked()
|
||||
}
|
||||
drained := c.drained
|
||||
c.mu.Unlock()
|
||||
|
||||
select {
|
||||
case <-drained:
|
||||
<-c.dispatcherDone
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("%w: %v", ErrChildRunDrainTimeout, ctx.Err())
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ChildRunAdmission) dispatch() {
|
||||
defer close(c.dispatcherDone)
|
||||
for {
|
||||
select {
|
||||
case <-c.wake:
|
||||
c.dispatchAvailable()
|
||||
case <-c.stop:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ChildRunAdmission) dispatchAvailable() {
|
||||
for {
|
||||
c.mu.Lock()
|
||||
request := c.oldestEligibleLocked()
|
||||
if request == nil {
|
||||
c.maybeDrainedLocked()
|
||||
c.mu.Unlock()
|
||||
return
|
||||
}
|
||||
launch := request.launch
|
||||
ticket := request.ticket
|
||||
frame := request.frame
|
||||
granted := c.grantLocked(request)
|
||||
c.mu.Unlock()
|
||||
|
||||
if launch && granted {
|
||||
go c.executeTop(ticket, frame)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ChildRunAdmission) oldestEligibleLocked() *childRunRequest {
|
||||
if c.closed || c.active >= c.processLimit {
|
||||
return nil
|
||||
}
|
||||
var oldest *childRunRequest
|
||||
for _, request := range c.pending {
|
||||
if request.resolved || !request.activated {
|
||||
continue
|
||||
}
|
||||
if request.frame.ctx.Err() != nil || !c.eligibleLocked(request.frame.constraints) {
|
||||
continue
|
||||
}
|
||||
if oldest == nil || request.sequence < oldest.sequence {
|
||||
oldest = request
|
||||
}
|
||||
}
|
||||
return oldest
|
||||
}
|
||||
|
||||
func (c *ChildRunAdmission) eligibleLocked(constraints ChildRunConstraints) bool {
|
||||
if c.active >= c.processLimit {
|
||||
return false
|
||||
}
|
||||
root := childRunRoot{tenant: constraints.TenantID, agent: constraints.RootAgentID}
|
||||
if constraints.RootAgentID != uuid.Nil && constraints.RootLimit > 0 &&
|
||||
c.rootActive[root] >= constraints.RootLimit {
|
||||
return false
|
||||
}
|
||||
parent := childRunParent{root: root, parent: constraints.ParentTaskID}
|
||||
return constraints.ParentTaskID == "" || constraints.ParentFanout <= 0 ||
|
||||
c.parentActive[parent] < constraints.ParentFanout
|
||||
}
|
||||
|
||||
func (c *ChildRunAdmission) grantLocked(request *childRunRequest) bool {
|
||||
if request.resolved {
|
||||
return false
|
||||
}
|
||||
if err := request.frame.ctx.Err(); err != nil {
|
||||
c.cancelRequestLocked(request, err)
|
||||
return false
|
||||
}
|
||||
c.removePendingLocked(request)
|
||||
request.resolved = true
|
||||
request.frame.state = ChildRunRunning
|
||||
request.ticket.current = request.frame
|
||||
c.running[request.ticket] = request.frame
|
||||
c.incrementLocked(request.frame.constraints)
|
||||
|
||||
if request.launch {
|
||||
request.ticket.state = ChildRunRunning
|
||||
request.ticket.live = true
|
||||
c.live++
|
||||
c.liveTickets[request.ticket] = struct{}{}
|
||||
close(request.ticket.started)
|
||||
} else if request.frame == request.ticket.top {
|
||||
request.ticket.state = ChildRunRunning
|
||||
} else {
|
||||
request.ticket.state = ChildRunWaitingChild
|
||||
}
|
||||
close(request.granted)
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *ChildRunAdmission) incrementLocked(constraints ChildRunConstraints) {
|
||||
c.active++
|
||||
root := childRunRoot{tenant: constraints.TenantID, agent: constraints.RootAgentID}
|
||||
if constraints.RootAgentID != uuid.Nil {
|
||||
c.rootActive[root]++
|
||||
}
|
||||
if constraints.ParentTaskID != "" {
|
||||
c.parentActive[childRunParent{root: root, parent: constraints.ParentTaskID}]++
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ChildRunAdmission) releaseActiveLocked(ticket *ChildRunTicket, frame *childRunFrame) {
|
||||
if c.running[ticket] != frame {
|
||||
return
|
||||
}
|
||||
delete(c.running, ticket)
|
||||
ticket.current = nil
|
||||
c.active--
|
||||
|
||||
root := childRunRoot{
|
||||
tenant: frame.constraints.TenantID,
|
||||
agent: frame.constraints.RootAgentID,
|
||||
}
|
||||
if frame.constraints.RootAgentID != uuid.Nil {
|
||||
decrementChildRunCount(c.rootActive, root)
|
||||
}
|
||||
if frame.constraints.ParentTaskID != "" {
|
||||
decrementChildRunCount(
|
||||
c.parentActive,
|
||||
childRunParent{root: root, parent: frame.constraints.ParentTaskID},
|
||||
)
|
||||
}
|
||||
c.signalLocked()
|
||||
}
|
||||
|
||||
func (c *ChildRunAdmission) executeTop(ticket *ChildRunTicket, frame *childRunFrame) {
|
||||
var callbackErr error
|
||||
defer func() {
|
||||
if recovered := recover(); recovered != nil {
|
||||
callbackErr = fmt.Errorf("child run panic: %v", recovered)
|
||||
}
|
||||
c.finishTop(ticket, callbackErr)
|
||||
}()
|
||||
frame.run(frame.ctx, frame.lease)
|
||||
}
|
||||
|
||||
func invokeChildRun(frame *childRunFrame) (callbackErr error) {
|
||||
defer func() {
|
||||
if recovered := recover(); recovered != nil {
|
||||
callbackErr = fmt.Errorf("child run panic: %v", recovered)
|
||||
}
|
||||
}()
|
||||
frame.run(frame.ctx, frame.lease)
|
||||
return callbackErr
|
||||
}
|
||||
|
||||
func (c *ChildRunAdmission) finishTop(ticket *ChildRunTicket, callbackErr error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if ticket.finished {
|
||||
return
|
||||
}
|
||||
|
||||
if current := c.running[ticket]; current != nil {
|
||||
c.releaseActiveLocked(ticket, current)
|
||||
}
|
||||
for request := range ticket.requests {
|
||||
c.cancelRequestLocked(request, terminalChildRunError(ticket))
|
||||
}
|
||||
for _, frame := range ticket.stack {
|
||||
if frame.stopCancel != nil {
|
||||
frame.stopCancel()
|
||||
}
|
||||
frame.cancel()
|
||||
}
|
||||
if ticket.stopCancel != nil {
|
||||
ticket.stopCancel()
|
||||
}
|
||||
|
||||
switch {
|
||||
case ticket.cancelRequested:
|
||||
ticket.state = ChildRunCancelled
|
||||
if ticket.err == nil {
|
||||
ticket.err = context.Canceled
|
||||
}
|
||||
case callbackErr != nil:
|
||||
ticket.state = ChildRunFailed
|
||||
ticket.err = callbackErr
|
||||
default:
|
||||
ticket.state = ChildRunCompleted
|
||||
}
|
||||
ticket.top.state = ticket.state
|
||||
ticket.finished = true
|
||||
if ticket.live {
|
||||
ticket.live = false
|
||||
c.live--
|
||||
delete(c.liveTickets, ticket)
|
||||
}
|
||||
ticket.doneOnce.Do(func() { close(ticket.done) })
|
||||
c.maybeDrainedLocked()
|
||||
c.signalLocked()
|
||||
}
|
||||
|
||||
func (c *ChildRunAdmission) finishContinuation(
|
||||
child *childRunFrame,
|
||||
callbackErr error,
|
||||
) *childRunRequest {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
ticket := child.lease.ticket
|
||||
|
||||
if c.running[ticket] == child {
|
||||
c.releaseActiveLocked(ticket, child)
|
||||
}
|
||||
if child.stopCancel != nil {
|
||||
child.stopCancel()
|
||||
}
|
||||
wasCancelled := child.cancelRequested || child.ctx.Err() != nil
|
||||
contextErr := child.ctx.Err()
|
||||
child.cancel()
|
||||
|
||||
switch {
|
||||
case wasCancelled:
|
||||
child.state = ChildRunCancelled
|
||||
if callbackErr == nil {
|
||||
callbackErr = contextErr
|
||||
}
|
||||
case callbackErr != nil:
|
||||
child.state = ChildRunFailed
|
||||
default:
|
||||
child.state = ChildRunCompleted
|
||||
}
|
||||
|
||||
if len(ticket.stack) > 1 && ticket.stack[len(ticket.stack)-1] == child {
|
||||
ticket.stack = ticket.stack[:len(ticket.stack)-1]
|
||||
}
|
||||
if ticket.cancelRequested || ticket.finished || c.closed {
|
||||
return nil
|
||||
}
|
||||
parent := ticket.stack[len(ticket.stack)-1]
|
||||
if parent.ctx.Err() != nil {
|
||||
c.cancelTicketLocked(ticket, parent.ctx.Err())
|
||||
return nil
|
||||
}
|
||||
parent.state = ChildRunQueued
|
||||
ticket.state = ChildRunWaitingChild
|
||||
request := c.newContinuationRequestLocked(ticket, parent)
|
||||
c.pending = append(c.pending, request)
|
||||
c.signalLocked()
|
||||
return request
|
||||
}
|
||||
|
||||
func (c *ChildRunAdmission) validateContinuationLocked(
|
||||
lease *ChildRunLease,
|
||||
child ChildRunConstraints,
|
||||
) (ChildRunConstraints, error) {
|
||||
ticket := lease.ticket
|
||||
if c.closed {
|
||||
return child, ErrChildRunClosed
|
||||
}
|
||||
if ticket.finished || ticket.cancelRequested {
|
||||
return child, terminalChildRunError(ticket)
|
||||
}
|
||||
if c.running[ticket] != lease.frame || ticket.current != lease.frame ||
|
||||
lease.frame.state != ChildRunRunning {
|
||||
return child, ErrChildRunInvalidContinuation
|
||||
}
|
||||
parent := lease.frame.constraints
|
||||
if parent.TaskID == "" || child.TaskID == "" || child.ParentTaskID != parent.TaskID {
|
||||
return child, ErrChildRunInvalidContinuation
|
||||
}
|
||||
if len(ticket.stack) >= maxChildRunContinuationDepth {
|
||||
return child, ErrChildRunContinuationDepth
|
||||
}
|
||||
if child.TenantID != parent.TenantID {
|
||||
return child, ErrChildRunInvalidContinuation
|
||||
}
|
||||
if child.Depth != parent.Depth+1 {
|
||||
return child, ErrChildRunInvalidContinuation
|
||||
}
|
||||
if child.Depth > ticket.maxDepth {
|
||||
return child, ErrChildRunContinuationDepth
|
||||
}
|
||||
child.MaxDepth = ticket.maxDepth
|
||||
if parent.RootAgentID != uuid.Nil && child.RootAgentID == parent.RootAgentID {
|
||||
child.RootLimit = parent.RootLimit
|
||||
}
|
||||
return child, nil
|
||||
}
|
||||
|
||||
func (c *ChildRunAdmission) newFrameLocked(
|
||||
ticket *ChildRunTicket,
|
||||
ctx context.Context,
|
||||
constraints ChildRunConstraints,
|
||||
run func(context.Context, *ChildRunLease),
|
||||
) *childRunFrame {
|
||||
runCtx, cancel := context.WithCancel(ctx)
|
||||
frame := &childRunFrame{
|
||||
constraints: constraints,
|
||||
ctx: runCtx,
|
||||
cancel: cancel,
|
||||
run: run,
|
||||
state: ChildRunNew,
|
||||
}
|
||||
frame.lease = &ChildRunLease{controller: c, ticket: ticket, frame: frame}
|
||||
if ticket.top != nil {
|
||||
frame.stopCancel = context.AfterFunc(runCtx, func() {
|
||||
c.cancelContinuationFrame(ticket, frame)
|
||||
})
|
||||
}
|
||||
return frame
|
||||
}
|
||||
|
||||
func (c *ChildRunAdmission) newContinuationRequestLocked(
|
||||
ticket *ChildRunTicket,
|
||||
frame *childRunFrame,
|
||||
) *childRunRequest {
|
||||
request := &childRunRequest{
|
||||
ticket: ticket,
|
||||
frame: frame,
|
||||
sequence: ticket.sequence,
|
||||
activated: true,
|
||||
granted: make(chan struct{}),
|
||||
cancelled: make(chan struct{}),
|
||||
}
|
||||
ticket.requests[request] = struct{}{}
|
||||
return request
|
||||
}
|
||||
|
||||
func (c *ChildRunAdmission) cancelContinuationFrame(
|
||||
ticket *ChildRunTicket,
|
||||
frame *childRunFrame,
|
||||
) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if ticket.finished || frame.state == ChildRunCompleted ||
|
||||
frame.state == ChildRunFailed || frame.state == ChildRunCancelled {
|
||||
return
|
||||
}
|
||||
frame.cancelRequested = true
|
||||
frame.cancel()
|
||||
for request := range ticket.requests {
|
||||
if request.frame == frame {
|
||||
c.cancelRequestLocked(request, frame.ctx.Err())
|
||||
}
|
||||
}
|
||||
c.signalLocked()
|
||||
}
|
||||
|
||||
func (c *ChildRunAdmission) cancelTicketLocked(ticket *ChildRunTicket, err error) {
|
||||
if ticket.finished || ticket.cancelRequested {
|
||||
return
|
||||
}
|
||||
ticket.cancelRequested = true
|
||||
ticket.err = err
|
||||
ticket.state = ChildRunCancelled
|
||||
for _, frame := range ticket.stack {
|
||||
frame.cancelRequested = true
|
||||
frame.cancel()
|
||||
}
|
||||
for request := range ticket.requests {
|
||||
c.cancelRequestLocked(request, err)
|
||||
}
|
||||
|
||||
if !ticket.live {
|
||||
ticket.finished = true
|
||||
ticket.top.state = ChildRunCancelled
|
||||
if ticket.stopCancel != nil {
|
||||
ticket.stopCancel()
|
||||
}
|
||||
ticket.doneOnce.Do(func() { close(ticket.done) })
|
||||
}
|
||||
c.maybeDrainedLocked()
|
||||
c.signalLocked()
|
||||
}
|
||||
|
||||
func (c *ChildRunAdmission) cancelRequestLocked(request *childRunRequest, err error) {
|
||||
if request.resolved {
|
||||
return
|
||||
}
|
||||
c.removePendingLocked(request)
|
||||
request.resolved = true
|
||||
request.cancelErr = err
|
||||
request.frame.cancelRequested = true
|
||||
request.frame.state = ChildRunCancelled
|
||||
close(request.cancelled)
|
||||
}
|
||||
|
||||
func (c *ChildRunAdmission) removePendingLocked(request *childRunRequest) {
|
||||
for index, pending := range c.pending {
|
||||
if pending != request {
|
||||
continue
|
||||
}
|
||||
c.pending = append(c.pending[:index], c.pending[index+1:]...)
|
||||
if request.independent {
|
||||
c.pendingIndependent--
|
||||
}
|
||||
break
|
||||
}
|
||||
delete(request.ticket.requests, request)
|
||||
}
|
||||
|
||||
func (c *ChildRunAdmission) maybeDrainedLocked() {
|
||||
if !c.closed || c.active != 0 || c.live != 0 || len(c.pending) != 0 {
|
||||
return
|
||||
}
|
||||
c.drainedOnce.Do(func() { close(c.drained) })
|
||||
c.stopOnce.Do(func() { close(c.stop) })
|
||||
}
|
||||
|
||||
func (c *ChildRunAdmission) signalLocked() {
|
||||
select {
|
||||
case c.wake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func waitChildRunRequest(request *childRunRequest) error {
|
||||
select {
|
||||
case <-request.granted:
|
||||
return nil
|
||||
case <-request.cancelled:
|
||||
if request.cancelErr != nil {
|
||||
return request.cancelErr
|
||||
}
|
||||
return context.Canceled
|
||||
}
|
||||
}
|
||||
|
||||
func terminalChildRunError(ticket *ChildRunTicket) error {
|
||||
if ticket.err != nil {
|
||||
return ticket.err
|
||||
}
|
||||
if ticket.cancelRequested {
|
||||
return context.Canceled
|
||||
}
|
||||
return ErrChildRunClosed
|
||||
}
|
||||
|
||||
func sameRootConstraint(left, right ChildRunConstraints) bool {
|
||||
return left.TenantID == right.TenantID &&
|
||||
left.RootAgentID != uuid.Nil &&
|
||||
left.RootAgentID == right.RootAgentID &&
|
||||
left.RootLimit == right.RootLimit
|
||||
}
|
||||
|
||||
func decrementChildRunCount[K comparable](counts map[K]int, key K) {
|
||||
if counts[key] <= 1 {
|
||||
delete(counts, key)
|
||||
return
|
||||
}
|
||||
counts[key]--
|
||||
}
|
||||
@@ -0,0 +1,859 @@
|
||||
package childrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const childRunTestTimeout = 3 * time.Second
|
||||
|
||||
func TestChildRunAdmissionStateTransitionsAndDormantActivation(t *testing.T) {
|
||||
controller := NewChildRunAdmission(1, 2)
|
||||
closeChildRunController(t, controller)
|
||||
|
||||
blockFirst := make(chan struct{})
|
||||
first := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(context.Context, *ChildRunLease) {
|
||||
<-blockFirst
|
||||
})
|
||||
waitChildRunSignal(t, first.Started(), "first start")
|
||||
|
||||
var ran atomic.Bool
|
||||
second, err := controller.Enqueue(context.Background(), ChildRunConstraints{}, func(context.Context, *ChildRunLease) {
|
||||
ran.Store(true)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Enqueue second: %v", err)
|
||||
}
|
||||
if got := second.State(); got != ChildRunNew {
|
||||
t.Fatalf("state before activation = %q, want %q", got, ChildRunNew)
|
||||
}
|
||||
if ran.Load() {
|
||||
t.Fatal("callback started before activation")
|
||||
}
|
||||
if err := second.Activate(); err != nil {
|
||||
t.Fatalf("Activate second: %v", err)
|
||||
}
|
||||
if got := second.State(); got != ChildRunQueued {
|
||||
t.Fatalf("state while capacity is occupied = %q, want %q", got, ChildRunQueued)
|
||||
}
|
||||
|
||||
close(blockFirst)
|
||||
waitChildRunSignal(t, second.Started(), "second start")
|
||||
waitChildRunSignal(t, second.Done(), "second completion")
|
||||
if got := second.State(); got != ChildRunCompleted {
|
||||
t.Fatalf("terminal state = %q, want %q", got, ChildRunCompleted)
|
||||
}
|
||||
if !ran.Load() {
|
||||
t.Fatal("callback did not run after activation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChildRunAdmissionEnforcesProcessRootAndParentLimitsAtGrant(t *testing.T) {
|
||||
controller := NewChildRunAdmission(3, 8)
|
||||
closeChildRunController(t, controller)
|
||||
|
||||
tenant := uuid.New()
|
||||
root := uuid.New()
|
||||
firstRelease := make(chan struct{})
|
||||
firstConstraints := ChildRunConstraints{
|
||||
TenantID: tenant, RootAgentID: root, RootLimit: 2,
|
||||
TaskID: "first", ParentTaskID: "parent", ParentFanout: 1,
|
||||
}
|
||||
first := enqueueAndActivateChildRun(t, controller, firstConstraints, func(context.Context, *ChildRunLease) {
|
||||
<-firstRelease
|
||||
})
|
||||
secondConstraints := firstConstraints
|
||||
secondConstraints.TaskID = "second"
|
||||
second := enqueueAndActivateChildRun(t, controller, secondConstraints, func(context.Context, *ChildRunLease) {})
|
||||
waitChildRunSignal(t, first.Started(), "first sibling start")
|
||||
assertChildRunNotSignalled(t, second.Started(), "second sibling bypassed immediate-parent fan-out")
|
||||
|
||||
otherRoot := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{
|
||||
TenantID: tenant, RootAgentID: uuid.New(), RootLimit: 1, TaskID: "other",
|
||||
}, func(context.Context, *ChildRunLease) {})
|
||||
waitChildRunSignal(t, otherRoot.Done(), "independent root completion")
|
||||
|
||||
close(firstRelease)
|
||||
waitChildRunSignal(t, second.Done(), "second sibling completion")
|
||||
if stats := controller.Stats(); stats.Active != 0 {
|
||||
t.Fatalf("active after completion = %d, want 0", stats.Active)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChildRunAdmissionUsesOldestEligibleWithoutHeadOfLineBlocking(t *testing.T) {
|
||||
controller := NewChildRunAdmission(2, 8)
|
||||
closeChildRunController(t, controller)
|
||||
|
||||
tenant := uuid.New()
|
||||
rootA := uuid.New()
|
||||
rootB := uuid.New()
|
||||
blockA := make(chan struct{})
|
||||
activeA := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{
|
||||
TenantID: tenant, RootAgentID: rootA, RootLimit: 1, TaskID: "active-a",
|
||||
}, func(context.Context, *ChildRunLease) { <-blockA })
|
||||
blockedA := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{
|
||||
TenantID: tenant, RootAgentID: rootA, RootLimit: 1, TaskID: "blocked-a",
|
||||
}, func(context.Context, *ChildRunLease) {})
|
||||
eligibleB := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{
|
||||
TenantID: tenant, RootAgentID: rootB, RootLimit: 1, TaskID: "eligible-b",
|
||||
}, func(context.Context, *ChildRunLease) {})
|
||||
|
||||
waitChildRunSignal(t, activeA.Started(), "active root A start")
|
||||
waitChildRunSignal(t, eligibleB.Done(), "eligible root B completion")
|
||||
assertChildRunNotSignalled(t, blockedA.Started(), "root-blocked entry started early")
|
||||
close(blockA)
|
||||
waitChildRunSignal(t, blockedA.Done(), "blocked root A completion")
|
||||
}
|
||||
|
||||
func TestChildRunAdmissionPreservesFIFOAmongEligibleRequests(t *testing.T) {
|
||||
controller := NewChildRunAdmission(1, 4)
|
||||
closeChildRunController(t, controller)
|
||||
|
||||
activeRelease := make(chan struct{})
|
||||
active := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(context.Context, *ChildRunLease) {
|
||||
<-activeRelease
|
||||
})
|
||||
waitChildRunSignal(t, active.Started(), "active blocker start")
|
||||
|
||||
firstRelease := make(chan struct{})
|
||||
first := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(context.Context, *ChildRunLease) {
|
||||
<-firstRelease
|
||||
})
|
||||
second := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(context.Context, *ChildRunLease) {})
|
||||
close(activeRelease)
|
||||
waitChildRunSignal(t, first.Started(), "oldest eligible start")
|
||||
assertChildRunNotSignalled(t, second.Started(), "later eligible request overtook oldest request")
|
||||
close(firstRelease)
|
||||
waitChildRunSignal(t, second.Done(), "later eligible completion")
|
||||
}
|
||||
|
||||
func TestChildRunAdmissionEnforcesStandardAndPerRootCapsAcrossRootlessWork(t *testing.T) {
|
||||
controller := NewChildRunAdmission(32, 40)
|
||||
closeChildRunController(t, controller)
|
||||
|
||||
tenant := uuid.New()
|
||||
root := uuid.New()
|
||||
startedRoot := make(chan struct{}, 21)
|
||||
startedRootless := make(chan struct{}, 12)
|
||||
release := make(chan struct{})
|
||||
var tickets []*ChildRunTicket
|
||||
for index := range 21 {
|
||||
tickets = append(tickets, enqueueAndActivateChildRun(t, controller, ChildRunConstraints{
|
||||
TenantID: tenant, RootAgentID: root, RootLimit: 20,
|
||||
TaskID: "root-task-" + string(rune('a'+index)),
|
||||
}, func(context.Context, *ChildRunLease) {
|
||||
startedRoot <- struct{}{}
|
||||
<-release
|
||||
}))
|
||||
}
|
||||
for index := range 12 {
|
||||
tickets = append(tickets, enqueueAndActivateChildRun(t, controller, ChildRunConstraints{
|
||||
TaskID: "rootless-task-" + string(rune('a'+index)),
|
||||
}, func(context.Context, *ChildRunLease) {
|
||||
startedRootless <- struct{}{}
|
||||
<-release
|
||||
}))
|
||||
}
|
||||
|
||||
for range 20 {
|
||||
waitChildRunSignal(t, startedRoot, "root-scoped admitted callback")
|
||||
}
|
||||
for range 12 {
|
||||
waitChildRunSignal(t, startedRootless, "rootless admitted callback")
|
||||
}
|
||||
assertChildRunNotSignalled(t, startedRoot, "twenty-first same-root callback bypassed root cap")
|
||||
if stats := controller.Stats(); stats.Active != 32 {
|
||||
t.Fatalf("active at saturation = %d, want Standard cap 32", stats.Active)
|
||||
}
|
||||
|
||||
close(release)
|
||||
for _, ticket := range tickets {
|
||||
waitChildRunSignal(t, ticket.Done(), "saturated callback completion")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChildRunAdmissionQueueCapacityIsExactly128IndependentChains(t *testing.T) {
|
||||
controller := NewChildRunAdmission(1, 128)
|
||||
closeChildRunController(t, controller)
|
||||
|
||||
activeRelease := make(chan struct{})
|
||||
active := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(context.Context, *ChildRunLease) {
|
||||
<-activeRelease
|
||||
})
|
||||
waitChildRunSignal(t, active.Started(), "queue blocker start")
|
||||
|
||||
queued := make([]*ChildRunTicket, 0, 128)
|
||||
for index := range 128 {
|
||||
ticket, err := controller.Enqueue(context.Background(), ChildRunConstraints{}, func(context.Context, *ChildRunLease) {})
|
||||
if err != nil {
|
||||
t.Fatalf("Enqueue pending ticket %d: %v", index, err)
|
||||
}
|
||||
queued = append(queued, ticket)
|
||||
}
|
||||
if _, err := controller.Enqueue(context.Background(), ChildRunConstraints{}, func(context.Context, *ChildRunLease) {}); !errors.Is(err, ErrChildRunBusy) {
|
||||
t.Fatalf("129th pending Enqueue error = %v, want ErrChildRunBusy", err)
|
||||
}
|
||||
if stats := controller.Stats(); stats.Pending != 128 {
|
||||
t.Fatalf("pending at queue saturation = %d, want 128", stats.Pending)
|
||||
}
|
||||
for _, ticket := range queued {
|
||||
if !ticket.Cancel() {
|
||||
t.Fatal("Cancel dormant queued ticket returned false")
|
||||
}
|
||||
}
|
||||
close(activeRelease)
|
||||
}
|
||||
|
||||
func TestChildRunAdmissionBoundsIndependentQueueButContinuationReusesChain(t *testing.T) {
|
||||
controller := NewChildRunAdmission(1, 1)
|
||||
closeChildRunController(t, controller)
|
||||
|
||||
tenant := uuid.New()
|
||||
root := uuid.New()
|
||||
var orderMu sync.Mutex
|
||||
var order []string
|
||||
var completedChildLease *ChildRunLease
|
||||
parentStarted := make(chan struct{})
|
||||
continueParent := make(chan struct{})
|
||||
|
||||
parent := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{
|
||||
TenantID: tenant, RootAgentID: root, RootLimit: 1,
|
||||
TaskID: "parent", Depth: 1, MaxDepth: 3,
|
||||
}, func(ctx context.Context, lease *ChildRunLease) {
|
||||
close(parentStarted)
|
||||
<-continueParent
|
||||
err := lease.Continue(ctx, ChildRunConstraints{
|
||||
TenantID: tenant, RootAgentID: root, RootLimit: 0,
|
||||
TaskID: "child", ParentTaskID: "parent", ParentFanout: 1,
|
||||
Depth: 2, MaxDepth: 3,
|
||||
}, func(_ context.Context, childLease *ChildRunLease) {
|
||||
completedChildLease = childLease
|
||||
if stats := controller.Stats(); stats.Active != 1 {
|
||||
t.Errorf("active during in-place child handoff = %d, want 1", stats.Active)
|
||||
}
|
||||
controller.mu.Lock()
|
||||
rootCount := controller.rootActive[childRunRoot{tenant: tenant, agent: root}]
|
||||
parentCount := controller.parentActive[childRunParent{
|
||||
root: childRunRoot{tenant: tenant, agent: root},
|
||||
parent: "parent",
|
||||
}]
|
||||
controller.mu.Unlock()
|
||||
if rootCount != 1 || parentCount != 1 {
|
||||
t.Errorf("handoff counters root=%d parent=%d, want 1/1", rootCount, parentCount)
|
||||
}
|
||||
orderMu.Lock()
|
||||
order = append(order, "child")
|
||||
orderMu.Unlock()
|
||||
})
|
||||
if err != nil {
|
||||
t.Errorf("Continue: %v", err)
|
||||
return
|
||||
}
|
||||
orderMu.Lock()
|
||||
order = append(order, "parent")
|
||||
orderMu.Unlock()
|
||||
})
|
||||
waitChildRunSignal(t, parentStarted, "parent start")
|
||||
|
||||
independent := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{
|
||||
TaskID: "independent",
|
||||
}, func(context.Context, *ChildRunLease) {
|
||||
orderMu.Lock()
|
||||
order = append(order, "independent")
|
||||
orderMu.Unlock()
|
||||
})
|
||||
if _, err := controller.Enqueue(context.Background(), ChildRunConstraints{}, func(context.Context, *ChildRunLease) {}); !errors.Is(err, ErrChildRunBusy) {
|
||||
t.Fatalf("second independent pending Enqueue error = %v, want ErrChildRunBusy", err)
|
||||
}
|
||||
|
||||
close(continueParent)
|
||||
waitChildRunSignal(t, parent.Done(), "parent continuation completion")
|
||||
waitChildRunSignal(t, independent.Done(), "independent completion")
|
||||
if completedChildLease == nil || completedChildLease.frame.state != ChildRunCompleted {
|
||||
t.Fatalf("normal continuation state = %v, want %q", completedChildLease, ChildRunCompleted)
|
||||
}
|
||||
orderMu.Lock()
|
||||
defer orderMu.Unlock()
|
||||
want := []string{"child", "parent", "independent"}
|
||||
if len(order) != len(want) {
|
||||
t.Fatalf("execution order = %v, want %v", order, want)
|
||||
}
|
||||
for index := range want {
|
||||
if order[index] != want[index] {
|
||||
t.Fatalf("execution order = %v, want %v", order, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChildRunAdmissionContinuationValidatesLineageAndDepth(t *testing.T) {
|
||||
controller := NewChildRunAdmission(1, 4)
|
||||
closeChildRunController(t, controller)
|
||||
|
||||
result := make(chan error, 4)
|
||||
ticket := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{
|
||||
TaskID: "parent", Depth: 1, MaxDepth: 1,
|
||||
}, func(ctx context.Context, lease *ChildRunLease) {
|
||||
result <- lease.Continue(ctx, ChildRunConstraints{
|
||||
TaskID: "wrong-lineage", ParentTaskID: "someone-else", Depth: 2, MaxDepth: 2,
|
||||
}, func(context.Context, *ChildRunLease) {})
|
||||
result <- lease.Continue(ctx, ChildRunConstraints{
|
||||
TaskID: "too-deep", ParentTaskID: "parent", Depth: 2, MaxDepth: 1,
|
||||
}, func(context.Context, *ChildRunLease) {})
|
||||
result <- lease.Continue(ctx, ChildRunConstraints{
|
||||
TaskID: "zero-depth", ParentTaskID: "parent", Depth: 0,
|
||||
}, func(context.Context, *ChildRunLease) {})
|
||||
result <- lease.Continue(ctx, ChildRunConstraints{
|
||||
TenantID: uuid.New(), TaskID: "other-tenant", ParentTaskID: "parent", Depth: 2,
|
||||
}, func(context.Context, *ChildRunLease) {})
|
||||
})
|
||||
waitChildRunSignal(t, ticket.Done(), "lineage validation callback completion")
|
||||
|
||||
if err := waitChildRunValue(t, result, "invalid lineage result"); !errors.Is(err, ErrChildRunInvalidContinuation) {
|
||||
t.Fatalf("invalid lineage error = %v, want ErrChildRunInvalidContinuation", err)
|
||||
}
|
||||
if err := waitChildRunValue(t, result, "depth result"); !errors.Is(err, ErrChildRunContinuationDepth) {
|
||||
t.Fatalf("depth error = %v, want ErrChildRunContinuationDepth", err)
|
||||
}
|
||||
if err := waitChildRunValue(t, result, "zero-depth result"); !errors.Is(err, ErrChildRunInvalidContinuation) {
|
||||
t.Fatalf("zero-depth error = %v, want ErrChildRunInvalidContinuation", err)
|
||||
}
|
||||
if err := waitChildRunValue(t, result, "tenant transition result"); !errors.Is(err, ErrChildRunInvalidContinuation) {
|
||||
t.Fatalf("tenant transition error = %v, want ErrChildRunInvalidContinuation", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChildRunAdmissionContinuationPanicResumesParent(t *testing.T) {
|
||||
controller := NewChildRunAdmission(1, 4)
|
||||
closeChildRunController(t, controller)
|
||||
|
||||
var resumed atomic.Bool
|
||||
var continuationErr error
|
||||
var failedChildLease *ChildRunLease
|
||||
parent := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{
|
||||
TaskID: "parent", Depth: 1, MaxDepth: 3,
|
||||
}, func(ctx context.Context, lease *ChildRunLease) {
|
||||
continuationErr = lease.Continue(ctx, ChildRunConstraints{
|
||||
TaskID: "child", ParentTaskID: "parent", Depth: 2, MaxDepth: 3,
|
||||
}, func(_ context.Context, childLease *ChildRunLease) {
|
||||
failedChildLease = childLease
|
||||
panic("nested boom")
|
||||
})
|
||||
resumed.Store(true)
|
||||
})
|
||||
waitChildRunSignal(t, parent.Done(), "parent resume after nested panic")
|
||||
if !resumed.Load() {
|
||||
t.Fatal("parent did not resume after nested callback panic")
|
||||
}
|
||||
if continuationErr == nil {
|
||||
t.Fatal("nested callback panic did not return an error")
|
||||
}
|
||||
if failedChildLease == nil || failedChildLease.frame.state != ChildRunFailed {
|
||||
t.Fatalf("panicking continuation state = %v, want %q", failedChildLease, ChildRunFailed)
|
||||
}
|
||||
if got := parent.State(); got != ChildRunCompleted {
|
||||
t.Fatalf("parent state after handled nested panic = %q, want %q", got, ChildRunCompleted)
|
||||
}
|
||||
if stats := controller.Stats(); stats.Active != 0 || stats.Pending != 0 {
|
||||
t.Fatalf("stats after nested panic = %+v, want zero active/pending", stats)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChildRunAdmissionQueuedContinuationContextCancellationResumesParent(t *testing.T) {
|
||||
controller := NewChildRunAdmission(2, 4)
|
||||
closeChildRunController(t, controller)
|
||||
|
||||
tenant := uuid.New()
|
||||
rootA := uuid.New()
|
||||
rootB := uuid.New()
|
||||
rootBRelease := make(chan struct{})
|
||||
rootBOwner := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{
|
||||
TenantID: tenant, RootAgentID: rootB, RootLimit: 1, TaskID: "root-b-owner",
|
||||
}, func(context.Context, *ChildRunLease) {
|
||||
<-rootBRelease
|
||||
})
|
||||
waitChildRunSignal(t, rootBOwner.Started(), "root B owner start")
|
||||
|
||||
childCtx, cancelChild := context.WithCancel(context.Background())
|
||||
var childRan atomic.Bool
|
||||
var continuationErr error
|
||||
var parentResumed atomic.Bool
|
||||
parent := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{
|
||||
TenantID: tenant, RootAgentID: rootA, RootLimit: 1,
|
||||
TaskID: "parent-a", Depth: 1, MaxDepth: 3,
|
||||
}, func(_ context.Context, lease *ChildRunLease) {
|
||||
continuationErr = lease.Continue(childCtx, ChildRunConstraints{
|
||||
TenantID: tenant, RootAgentID: rootB, RootLimit: 1,
|
||||
TaskID: "child-b", ParentTaskID: "parent-a", ParentFanout: 1,
|
||||
Depth: 2, MaxDepth: 3,
|
||||
}, func(context.Context, *ChildRunLease) {
|
||||
childRan.Store(true)
|
||||
})
|
||||
parentResumed.Store(true)
|
||||
})
|
||||
waitChildRunSignal(t, parent.Started(), "parent A start")
|
||||
waitForChildRunCondition(t, "queued continuation state", func() bool {
|
||||
return parent.State() == ChildRunWaitingChild && controller.Stats().Pending == 1
|
||||
})
|
||||
cancelChild()
|
||||
waitChildRunSignal(t, parent.Done(), "parent resume after child context cancellation")
|
||||
if childRan.Load() {
|
||||
t.Fatal("cancelled queued continuation callback ran")
|
||||
}
|
||||
if !parentResumed.Load() {
|
||||
t.Fatal("parent did not resume after queued child context cancellation")
|
||||
}
|
||||
if !errors.Is(continuationErr, context.Canceled) {
|
||||
t.Fatalf("continuation error = %v, want context.Canceled", continuationErr)
|
||||
}
|
||||
close(rootBRelease)
|
||||
}
|
||||
|
||||
func TestChildRunAdmissionSerializesConcurrentSynchronousContinuations(t *testing.T) {
|
||||
controller := NewChildRunAdmission(1, 4)
|
||||
closeChildRunController(t, controller)
|
||||
|
||||
tenant := uuid.New()
|
||||
root := uuid.New()
|
||||
var active atomic.Int32
|
||||
var maximum atomic.Int32
|
||||
var calls atomic.Int32
|
||||
childStarted := make(chan struct{}, 2)
|
||||
releaseChildren := make(chan struct{})
|
||||
|
||||
parent := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{
|
||||
TenantID: tenant, RootAgentID: root, RootLimit: 1,
|
||||
TaskID: "parent", Depth: 1, MaxDepth: 3,
|
||||
}, func(ctx context.Context, lease *ChildRunLease) {
|
||||
var group sync.WaitGroup
|
||||
for index := range 2 {
|
||||
index := index
|
||||
group.Go(func() {
|
||||
err := lease.Continue(ctx, ChildRunConstraints{
|
||||
TenantID: tenant, RootAgentID: root, RootLimit: 1,
|
||||
TaskID: "child-" + string(rune('a'+index)), ParentTaskID: "parent",
|
||||
ParentFanout: 1, Depth: 2, MaxDepth: 3,
|
||||
}, func(context.Context, *ChildRunLease) {
|
||||
current := active.Add(1)
|
||||
for {
|
||||
observed := maximum.Load()
|
||||
if current <= observed || maximum.CompareAndSwap(observed, current) {
|
||||
break
|
||||
}
|
||||
}
|
||||
calls.Add(1)
|
||||
childStarted <- struct{}{}
|
||||
<-releaseChildren
|
||||
active.Add(-1)
|
||||
})
|
||||
if err != nil {
|
||||
t.Errorf("Continue %d: %v", index, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
waitChildRunSignal(t, childStarted, "first serialized child start")
|
||||
assertChildRunNotSignalled(t, childStarted, "second synchronous child overlapped first")
|
||||
close(releaseChildren)
|
||||
group.Wait()
|
||||
})
|
||||
waitChildRunSignal(t, parent.Done(), "serialized continuation parent completion")
|
||||
if got := calls.Load(); got != 2 {
|
||||
t.Fatalf("child callback count = %d, want 2", got)
|
||||
}
|
||||
if got := maximum.Load(); got != 1 {
|
||||
t.Fatalf("maximum concurrent synchronous children = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChildRunAdmissionConstraintChangingContinuationReleasesParentCapacity(t *testing.T) {
|
||||
controller := NewChildRunAdmission(2, 8)
|
||||
closeChildRunController(t, controller)
|
||||
|
||||
tenant := uuid.New()
|
||||
rootA := uuid.New()
|
||||
rootB := uuid.New()
|
||||
rootBRelease := make(chan struct{})
|
||||
rootBOwner := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{
|
||||
TenantID: tenant, RootAgentID: rootB, RootLimit: 1, TaskID: "root-b-owner",
|
||||
}, func(context.Context, *ChildRunLease) {
|
||||
<-rootBRelease
|
||||
})
|
||||
waitChildRunSignal(t, rootBOwner.Started(), "root B owner start")
|
||||
|
||||
parentEntered := make(chan struct{})
|
||||
childBStarted := make(chan struct{})
|
||||
parentResumed := make(chan struct{})
|
||||
parent := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{
|
||||
TenantID: tenant, RootAgentID: rootA, RootLimit: 1,
|
||||
TaskID: "parent-a", Depth: 1, MaxDepth: 3,
|
||||
}, func(ctx context.Context, lease *ChildRunLease) {
|
||||
close(parentEntered)
|
||||
err := lease.Continue(ctx, ChildRunConstraints{
|
||||
TenantID: tenant, RootAgentID: rootB, RootLimit: 1,
|
||||
TaskID: "child-b", ParentTaskID: "parent-a", ParentFanout: 1,
|
||||
Depth: 2, MaxDepth: 3,
|
||||
}, func(context.Context, *ChildRunLease) {
|
||||
close(childBStarted)
|
||||
})
|
||||
if err != nil {
|
||||
t.Errorf("Continue to root B: %v", err)
|
||||
return
|
||||
}
|
||||
close(parentResumed)
|
||||
})
|
||||
waitChildRunSignal(t, parentEntered, "root A parent start")
|
||||
|
||||
unrelated := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{
|
||||
TenantID: tenant, RootAgentID: uuid.New(), RootLimit: 1, TaskID: "unrelated",
|
||||
}, func(context.Context, *ChildRunLease) {})
|
||||
waitChildRunSignal(t, unrelated.Done(), "unrelated work while parent is suspended")
|
||||
assertChildRunNotSignalled(t, childBStarted, "root B child bypassed saturated root")
|
||||
|
||||
close(rootBRelease)
|
||||
waitChildRunSignal(t, childBStarted, "root B child start")
|
||||
waitChildRunSignal(t, parentResumed, "root A parent resume")
|
||||
waitChildRunSignal(t, parent.Done(), "root A parent completion")
|
||||
}
|
||||
|
||||
func TestChildRunAdmissionCancellationBeforeGrantRemovesWaiter(t *testing.T) {
|
||||
controller := NewChildRunAdmission(1, 2)
|
||||
closeChildRunController(t, controller)
|
||||
|
||||
block := make(chan struct{})
|
||||
active := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(context.Context, *ChildRunLease) {
|
||||
<-block
|
||||
})
|
||||
waitChildRunSignal(t, active.Started(), "active run start")
|
||||
|
||||
var ran atomic.Bool
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
queued, err := controller.Enqueue(ctx, ChildRunConstraints{}, func(context.Context, *ChildRunLease) {
|
||||
ran.Store(true)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Enqueue queued: %v", err)
|
||||
}
|
||||
if err := queued.Activate(); err != nil {
|
||||
t.Fatalf("Activate queued: %v", err)
|
||||
}
|
||||
cancel()
|
||||
waitChildRunSignal(t, queued.Done(), "queued cancellation")
|
||||
if got := queued.State(); got != ChildRunCancelled {
|
||||
t.Fatalf("queued state = %q, want %q", got, ChildRunCancelled)
|
||||
}
|
||||
if ran.Load() {
|
||||
t.Fatal("cancelled queued callback ran")
|
||||
}
|
||||
if stats := controller.Stats(); stats.Pending != 0 {
|
||||
t.Fatalf("pending after cancellation = %d, want 0", stats.Pending)
|
||||
}
|
||||
close(block)
|
||||
}
|
||||
|
||||
func TestChildRunAdmissionGrantWinningCancellationRetainsLeaseUntilReturn(t *testing.T) {
|
||||
controller := NewChildRunAdmission(1, 2)
|
||||
closeChildRunController(t, controller)
|
||||
|
||||
callbackStarted := make(chan struct{})
|
||||
callbackRelease := make(chan struct{})
|
||||
var contextCancelled atomic.Bool
|
||||
first := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(ctx context.Context, lease *ChildRunLease) {
|
||||
close(callbackStarted)
|
||||
<-ctx.Done()
|
||||
contextCancelled.Store(true)
|
||||
<-callbackRelease
|
||||
})
|
||||
waitChildRunSignal(t, callbackStarted, "cancellation-resistant callback start")
|
||||
|
||||
second := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(context.Context, *ChildRunLease) {})
|
||||
if !first.Cancel() {
|
||||
t.Fatal("Cancel running ticket returned false")
|
||||
}
|
||||
if got := first.State(); got != ChildRunCancelled {
|
||||
t.Fatalf("running cancellation state = %q, want %q", got, ChildRunCancelled)
|
||||
}
|
||||
waitForChildRunCondition(t, "callback context cancellation", contextCancelled.Load)
|
||||
if stats := controller.Stats(); stats.Active != 1 {
|
||||
t.Fatalf("active after early Release calls = %d, want 1", stats.Active)
|
||||
}
|
||||
assertChildRunNotSignalled(t, second.Started(), "second run started before cancelled callback returned")
|
||||
|
||||
close(callbackRelease)
|
||||
waitChildRunSignal(t, first.Done(), "cancelled callback return")
|
||||
waitChildRunSignal(t, second.Done(), "second callback completion")
|
||||
}
|
||||
|
||||
func TestChildRunAdmissionLeaseReleaseReturnsCapacityBeforeCallbackReturn(t *testing.T) {
|
||||
controller := NewChildRunAdmission(1, 2)
|
||||
closeChildRunController(t, controller)
|
||||
|
||||
callbackStarted := make(chan struct{})
|
||||
leaseReleased := make(chan struct{})
|
||||
callbackRelease := make(chan struct{})
|
||||
first := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(_ context.Context, lease *ChildRunLease) {
|
||||
close(callbackStarted)
|
||||
lease.Release()
|
||||
lease.Release()
|
||||
close(leaseReleased)
|
||||
<-callbackRelease
|
||||
})
|
||||
waitChildRunSignal(t, callbackStarted, "first callback start")
|
||||
waitChildRunSignal(t, leaseReleased, "lease release")
|
||||
if stats := controller.Stats(); stats.Active != 0 {
|
||||
t.Fatalf("active after Release calls = %d, want 0", stats.Active)
|
||||
}
|
||||
|
||||
second := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(context.Context, *ChildRunLease) {})
|
||||
waitChildRunSignal(t, second.Done(), "second run after explicit release")
|
||||
if stats := controller.Stats(); stats.Live != 1 {
|
||||
t.Fatalf("live tickets while first callback remains open = %d, want 1", stats.Live)
|
||||
}
|
||||
assertChildRunNotSignalled(t, first.Done(), "ticket completed before callback return")
|
||||
|
||||
close(callbackRelease)
|
||||
waitChildRunSignal(t, first.Done(), "first callback return")
|
||||
}
|
||||
|
||||
func TestChildRunAdmissionCloseCancelsReleasedLiveCallbackAndDrains(t *testing.T) {
|
||||
controller := NewChildRunAdmission(1, 2)
|
||||
|
||||
callbackStarted := make(chan struct{})
|
||||
leaseReleased := make(chan struct{})
|
||||
callbackCancelled := make(chan struct{})
|
||||
ticket := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(ctx context.Context, lease *ChildRunLease) {
|
||||
close(callbackStarted)
|
||||
lease.Release()
|
||||
close(leaseReleased)
|
||||
<-ctx.Done()
|
||||
close(callbackCancelled)
|
||||
})
|
||||
waitChildRunSignal(t, callbackStarted, "callback start")
|
||||
waitChildRunSignal(t, leaseReleased, "lease release")
|
||||
|
||||
if stats := controller.Stats(); stats.Active != 0 || stats.Live != 1 {
|
||||
t.Fatalf("stats after lease release = %+v, want active=0 live=1", stats)
|
||||
}
|
||||
|
||||
closeCtx, cancelClose := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancelClose()
|
||||
if err := controller.Close(closeCtx); err != nil {
|
||||
t.Fatalf("Close: %v", err)
|
||||
}
|
||||
|
||||
waitChildRunSignal(t, callbackCancelled, "callback cancellation")
|
||||
waitChildRunSignal(t, ticket.Done(), "ticket completion")
|
||||
if got := ticket.State(); got != ChildRunCancelled {
|
||||
t.Fatalf("ticket state = %q, want %q", got, ChildRunCancelled)
|
||||
}
|
||||
if err := ticket.Err(); !errors.Is(err, ErrChildRunClosed) {
|
||||
t.Fatalf("ticket error = %v, want ErrChildRunClosed", err)
|
||||
}
|
||||
if stats := controller.Stats(); !stats.Closed || stats.Active != 0 || stats.Pending != 0 || stats.Live != 0 {
|
||||
t.Fatalf("final stats = %+v, want closed and drained", stats)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChildRunAdmissionCancellationDuringTransferredChildReleasesExactlyOnce(t *testing.T) {
|
||||
controller := NewChildRunAdmission(1, 4)
|
||||
closeChildRunController(t, controller)
|
||||
|
||||
tenant := uuid.New()
|
||||
root := uuid.New()
|
||||
childStarted := make(chan struct{})
|
||||
childRelease := make(chan struct{})
|
||||
parent := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{
|
||||
TenantID: tenant, RootAgentID: root, RootLimit: 1,
|
||||
TaskID: "parent", Depth: 1, MaxDepth: 3,
|
||||
}, func(ctx context.Context, lease *ChildRunLease) {
|
||||
_ = lease.Continue(ctx, ChildRunConstraints{
|
||||
TenantID: tenant, RootAgentID: root, RootLimit: 1,
|
||||
TaskID: "child", ParentTaskID: "parent", ParentFanout: 1,
|
||||
Depth: 2, MaxDepth: 3,
|
||||
}, func(context.Context, *ChildRunLease) {
|
||||
close(childStarted)
|
||||
<-childRelease
|
||||
})
|
||||
})
|
||||
waitChildRunSignal(t, childStarted, "transferred child start")
|
||||
if !parent.Cancel() {
|
||||
t.Fatal("Cancel parent during child returned false")
|
||||
}
|
||||
if stats := controller.Stats(); stats.Active != 1 {
|
||||
t.Fatalf("active while cancellation-resistant child runs = %d, want 1", stats.Active)
|
||||
}
|
||||
close(childRelease)
|
||||
waitChildRunSignal(t, parent.Done(), "cancelled continuation chain completion")
|
||||
if stats := controller.Stats(); stats.Active != 0 || stats.Pending != 0 {
|
||||
t.Fatalf("stats after cancelled continuation = %+v, want zero active/pending", stats)
|
||||
}
|
||||
controller.mu.Lock()
|
||||
defer controller.mu.Unlock()
|
||||
if len(controller.rootActive) != 0 || len(controller.parentActive) != 0 {
|
||||
t.Fatalf("constraint counters leaked: root=%v parent=%v", controller.rootActive, controller.parentActive)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChildRunAdmissionPanicReleasesCapacityAndMarksFailed(t *testing.T) {
|
||||
controller := NewChildRunAdmission(1, 2)
|
||||
closeChildRunController(t, controller)
|
||||
|
||||
failed := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(context.Context, *ChildRunLease) {
|
||||
panic("boom")
|
||||
})
|
||||
next := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(context.Context, *ChildRunLease) {})
|
||||
waitChildRunSignal(t, failed.Done(), "panicking callback completion")
|
||||
if got := failed.State(); got != ChildRunFailed {
|
||||
t.Fatalf("panic state = %q, want %q", got, ChildRunFailed)
|
||||
}
|
||||
if failed.Err() == nil {
|
||||
t.Fatal("panic did not record an error")
|
||||
}
|
||||
waitChildRunSignal(t, next.Done(), "post-panic callback completion")
|
||||
}
|
||||
|
||||
func TestChildRunAdmissionCloseIsIdempotentDrainsAndStopsDispatcher(t *testing.T) {
|
||||
controller := NewChildRunAdmission(1, 2)
|
||||
callbackStarted := make(chan struct{})
|
||||
callbackRelease := make(chan struct{})
|
||||
active := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(context.Context, *ChildRunLease) {
|
||||
close(callbackStarted)
|
||||
<-callbackRelease
|
||||
})
|
||||
waitChildRunSignal(t, callbackStarted, "active callback start")
|
||||
|
||||
var queuedRan atomic.Bool
|
||||
queued := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(context.Context, *ChildRunLease) {
|
||||
queuedRan.Store(true)
|
||||
})
|
||||
closeCtx, cancelClose := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
||||
defer cancelClose()
|
||||
if err := controller.Close(closeCtx); !errors.Is(err, ErrChildRunDrainTimeout) {
|
||||
t.Fatalf("first Close error = %v, want ErrChildRunDrainTimeout", err)
|
||||
}
|
||||
waitChildRunSignal(t, queued.Done(), "queued close cancellation")
|
||||
if queuedRan.Load() {
|
||||
t.Fatal("queued callback ran during close")
|
||||
}
|
||||
if stats := controller.Stats(); stats.Active != 1 || !stats.Closed {
|
||||
t.Fatalf("stats after timed-out close = %+v", stats)
|
||||
}
|
||||
if _, err := controller.Enqueue(context.Background(), ChildRunConstraints{}, func(context.Context, *ChildRunLease) {}); !errors.Is(err, ErrChildRunClosed) {
|
||||
t.Fatalf("Enqueue after Close error = %v, want ErrChildRunClosed", err)
|
||||
}
|
||||
|
||||
close(callbackRelease)
|
||||
waitChildRunSignal(t, active.Done(), "active callback drain")
|
||||
if err := controller.Close(context.Background()); err != nil {
|
||||
t.Fatalf("second Close: %v", err)
|
||||
}
|
||||
waitChildRunSignal(t, controller.dispatcherDone, "dispatcher stop")
|
||||
if err := controller.Close(context.Background()); err != nil {
|
||||
t.Fatalf("third Close: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChildRunAdmissionConcurrentCloseCallsShareOneDrain(t *testing.T) {
|
||||
controller := NewChildRunAdmission(1, 1)
|
||||
callbackStarted := make(chan struct{})
|
||||
callbackRelease := make(chan struct{})
|
||||
ticket := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(context.Context, *ChildRunLease) {
|
||||
close(callbackStarted)
|
||||
<-callbackRelease
|
||||
})
|
||||
waitChildRunSignal(t, callbackStarted, "callback start before concurrent close")
|
||||
|
||||
closeResults := make(chan error, 2)
|
||||
startClose := make(chan struct{})
|
||||
for range 2 {
|
||||
go func() {
|
||||
<-startClose
|
||||
closeResults <- controller.Close(context.Background())
|
||||
}()
|
||||
}
|
||||
close(startClose)
|
||||
waitForChildRunCondition(t, "controller closed by concurrent caller", func() bool {
|
||||
return controller.Stats().Closed
|
||||
})
|
||||
close(callbackRelease)
|
||||
waitChildRunSignal(t, ticket.Done(), "callback drain under concurrent close")
|
||||
for index := range 2 {
|
||||
if err := waitChildRunValue(t, closeResults, "concurrent Close result"); err != nil {
|
||||
t.Fatalf("concurrent Close %d: %v", index, err)
|
||||
}
|
||||
}
|
||||
waitChildRunSignal(t, controller.dispatcherDone, "dispatcher stop after concurrent close")
|
||||
}
|
||||
|
||||
func enqueueAndActivateChildRun(
|
||||
t *testing.T,
|
||||
controller *ChildRunAdmission,
|
||||
constraints ChildRunConstraints,
|
||||
run func(context.Context, *ChildRunLease),
|
||||
) *ChildRunTicket {
|
||||
t.Helper()
|
||||
ticket, err := controller.Enqueue(context.Background(), constraints, run)
|
||||
if err != nil {
|
||||
t.Fatalf("Enqueue: %v", err)
|
||||
}
|
||||
if err := ticket.Activate(); err != nil {
|
||||
t.Fatalf("Activate: %v", err)
|
||||
}
|
||||
return ticket
|
||||
}
|
||||
|
||||
func closeChildRunController(t *testing.T, controller *ChildRunAdmission) {
|
||||
t.Helper()
|
||||
t.Cleanup(func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), childRunTestTimeout)
|
||||
defer cancel()
|
||||
if err := controller.Close(ctx); err != nil {
|
||||
t.Errorf("Close: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func waitChildRunSignal(t *testing.T, signal <-chan struct{}, description string) {
|
||||
t.Helper()
|
||||
select {
|
||||
case <-signal:
|
||||
case <-time.After(childRunTestTimeout):
|
||||
t.Fatalf("timed out waiting for %s", description)
|
||||
}
|
||||
}
|
||||
|
||||
func assertChildRunNotSignalled(t *testing.T, signal <-chan struct{}, description string) {
|
||||
t.Helper()
|
||||
select {
|
||||
case <-signal:
|
||||
t.Fatal(description)
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
|
||||
func waitChildRunValue[T any](t *testing.T, values <-chan T, description string) T {
|
||||
t.Helper()
|
||||
select {
|
||||
case value := <-values:
|
||||
return value
|
||||
case <-time.After(childRunTestTimeout):
|
||||
t.Fatalf("timed out waiting for %s", description)
|
||||
var zero T
|
||||
return zero
|
||||
}
|
||||
}
|
||||
|
||||
func waitForChildRunCondition(t *testing.T, description string, condition func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(childRunTestTimeout)
|
||||
for !condition() {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("timed out waiting for %s", description)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
}
|
||||
@@ -606,10 +606,10 @@ func (cc CronConfig) ToRetryConfig() cron.RetryConfig {
|
||||
return cfg
|
||||
}
|
||||
|
||||
// SubagentsConfig configures the subagent system (matching TS agents.defaults.subagents).
|
||||
// SubagentsConfig configures the GoClaw subagent system.
|
||||
// All fields optional — zero values mean "use default".
|
||||
type SubagentsConfig struct {
|
||||
MaxConcurrent int `json:"maxConcurrent,omitempty"` // default 8 (TS: DEFAULT_SUBAGENT_MAX_CONCURRENT)
|
||||
MaxConcurrent int `json:"maxConcurrent,omitempty"` // executing descendants per root agent; default 20
|
||||
MaxSpawnDepth int `json:"maxSpawnDepth,omitempty"` // default 1, range 1-5
|
||||
MaxChildrenPerAgent int `json:"maxChildrenPerAgent,omitempty"` // default 5, range 1-20
|
||||
ArchiveAfterMinutes int `json:"archiveAfterMinutes,omitempty"` // default 60
|
||||
|
||||
@@ -16,10 +16,10 @@ type Edition struct {
|
||||
MaxSubagentDepth int `json:"max_subagent_depth"` // 0 = use config default
|
||||
KGEnabled bool `json:"kg_enabled"`
|
||||
RBACEnabled bool `json:"rbac_enabled"`
|
||||
TeamFullMode bool `json:"team_full_mode"` // false = lite task actions only
|
||||
VectorSearch bool `json:"vector_search"` // false = FTS5 only
|
||||
SupportsPipNpm bool `json:"supports_pip_npm"` // false for Lite desktop
|
||||
SupportsApk bool `json:"supports_apk"` // false for Lite desktop (no apk on macOS/Windows)
|
||||
TeamFullMode bool `json:"team_full_mode"` // false = lite task actions only
|
||||
VectorSearch bool `json:"vector_search"` // false = FTS5 only
|
||||
SupportsPipNpm bool `json:"supports_pip_npm"` // false for Lite desktop
|
||||
SupportsApk bool `json:"supports_apk"` // false for Lite desktop (no apk on macOS/Windows)
|
||||
}
|
||||
|
||||
// --- Presets ---
|
||||
@@ -93,3 +93,13 @@ func (e Edition) ChannelLimit(channelType string) int {
|
||||
func (e Edition) AllowsChannels() bool {
|
||||
return e.Name == "standard"
|
||||
}
|
||||
|
||||
// ChildRunLimit returns the process-wide cap for executing self-spawn and
|
||||
// Agent Link child callbacks. It is separate from the per-agent
|
||||
// maxConcurrent setting and is intentionally not serialized.
|
||||
func (e Edition) ChildRunLimit() int {
|
||||
if e.Name == Lite.Name {
|
||||
return 2
|
||||
}
|
||||
return 32
|
||||
}
|
||||
@@ -295,6 +295,15 @@ func TestLiteEditionFeatures(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestEditionChildRunLimit(t *testing.T) {
|
||||
if got := Standard.ChildRunLimit(); got != 32 {
|
||||
t.Fatalf("Standard.ChildRunLimit() = %d, want 32", got)
|
||||
}
|
||||
if got := Lite.ChildRunLimit(); got != 2 {
|
||||
t.Fatalf("Lite.ChildRunLimit() = %d, want 2", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLiteEditionChannelLimits verifies Lite channel constraints.
|
||||
func TestLiteEditionChannelLimits(t *testing.T) {
|
||||
e := Lite
|
||||
|
||||
@@ -88,3 +88,85 @@ func TestBridgeContextMiddleware_NoStore_NoAgentKey(t *testing.T) {
|
||||
t.Errorf("ToolAgentKeyFromCtx = %q, want empty when no agent store is wired", gotKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBridgeContextMiddleware_InjectsSignedDelegationArtifactContext(t *testing.T) {
|
||||
const (
|
||||
gatewayToken = "test-gateway-token"
|
||||
delegationID = "delegation-123"
|
||||
inputsRoot = "/runtime/delegations/delegation-123/inputs"
|
||||
)
|
||||
agentID := uuid.New()
|
||||
var gotDelegationID, gotInputs string
|
||||
var artifactRun bool
|
||||
next := http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
|
||||
gotDelegationID = tools.DelegationIDFromCtx(r.Context())
|
||||
gotInputs = tools.DelegationArtifactInputsFromCtx(r.Context())
|
||||
artifactRun = tools.IsDelegationArtifactRun(r.Context())
|
||||
})
|
||||
mw := bridgeContextMiddleware(gatewayToken, nil, next)
|
||||
|
||||
sig := providers.SignBridgeContext(
|
||||
gatewayToken,
|
||||
agentID.String(),
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"/runtime/delegations/delegation-123/outputs",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
delegationID,
|
||||
inputsRoot,
|
||||
)
|
||||
req := httptest.NewRequest(http.MethodPost, "/mcp/bridge", nil)
|
||||
req.Header.Set("X-Agent-ID", agentID.String())
|
||||
req.Header.Set("X-Workspace", "/runtime/delegations/delegation-123/outputs")
|
||||
req.Header.Set("X-Delegation-ID", delegationID)
|
||||
req.Header.Set("X-Delegation-Inputs", inputsRoot)
|
||||
req.Header.Set("X-Bridge-Sig", sig)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
mw.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK || !artifactRun {
|
||||
t.Fatalf("status=%d artifactRun=%v body=%s", rec.Code, artifactRun, rec.Body.String())
|
||||
}
|
||||
if gotDelegationID != delegationID || gotInputs != inputsRoot {
|
||||
t.Fatalf("delegation context = (%q, %q)", gotDelegationID, gotInputs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBridgeContextMiddleware_RejectsLegacySignatureWithDelegationHeaders(t *testing.T) {
|
||||
const gatewayToken = "test-gateway-token"
|
||||
agentID := uuid.New()
|
||||
legacySig := providers.SignBridgeContext(
|
||||
gatewayToken,
|
||||
agentID.String(),
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"/runtime/outputs",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
)
|
||||
req := httptest.NewRequest(http.MethodPost, "/mcp/bridge", nil)
|
||||
req.Header.Set("X-Agent-ID", agentID.String())
|
||||
req.Header.Set("X-Workspace", "/runtime/outputs")
|
||||
req.Header.Set("X-Delegation-ID", "delegation-123")
|
||||
req.Header.Set("X-Delegation-Inputs", "/runtime/inputs")
|
||||
req.Header.Set("X-Bridge-Sig", legacySig)
|
||||
|
||||
var called bool
|
||||
mw := bridgeContextMiddleware(gatewayToken, nil, http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
|
||||
called = true
|
||||
}))
|
||||
rec := httptest.NewRecorder()
|
||||
mw.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusForbidden || called {
|
||||
t.Fatalf("status=%d called=%v, want forbidden", rec.Code, called)
|
||||
}
|
||||
}
|
||||
@@ -373,6 +373,8 @@ func bridgeContextMiddleware(gatewayToken string, agentStore store.AgentStore, n
|
||||
workspace := r.Header.Get("X-Workspace")
|
||||
localKey := r.Header.Get("X-Local-Key")
|
||||
sessionKey := r.Header.Get("X-Session-Key")
|
||||
delegationID := r.Header.Get("X-Delegation-ID")
|
||||
delegationInputs := r.Header.Get("X-Delegation-Inputs")
|
||||
|
||||
if agentIDStr != "" || userID != "" {
|
||||
// Reject context headers when no gateway token — prevents unauthenticated impersonation.
|
||||
@@ -386,7 +388,44 @@ func bridgeContextMiddleware(gatewayToken string, agentStore store.AgentStore, n
|
||||
// Verify HMAC signature over all context fields.
|
||||
tenantIDStr := r.Header.Get("X-Tenant-ID")
|
||||
sig := r.Header.Get("X-Bridge-Sig")
|
||||
ok, tenantVerified := providers.VerifyBridgeContext(gatewayToken, agentIDStr, userID, channel, chatID, peerKind, workspace, tenantIDStr, sig, localKey, sessionKey)
|
||||
var ok, tenantVerified bool
|
||||
hasDelegationContext := delegationID != "" || delegationInputs != ""
|
||||
if hasDelegationContext {
|
||||
if delegationID == "" || delegationInputs == "" {
|
||||
http.Error(w, `{"error":"incomplete delegation bridge context"}`, http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
expected := providers.SignBridgeContext(
|
||||
gatewayToken,
|
||||
agentIDStr,
|
||||
userID,
|
||||
channel,
|
||||
chatID,
|
||||
peerKind,
|
||||
workspace,
|
||||
tenantIDStr,
|
||||
localKey,
|
||||
sessionKey,
|
||||
delegationID,
|
||||
delegationInputs,
|
||||
)
|
||||
ok = subtle.ConstantTimeCompare([]byte(sig), []byte(expected)) == 1
|
||||
tenantVerified = ok
|
||||
} else {
|
||||
ok, tenantVerified = providers.VerifyBridgeContext(
|
||||
gatewayToken,
|
||||
agentIDStr,
|
||||
userID,
|
||||
channel,
|
||||
chatID,
|
||||
peerKind,
|
||||
workspace,
|
||||
tenantIDStr,
|
||||
sig,
|
||||
localKey,
|
||||
sessionKey,
|
||||
)
|
||||
}
|
||||
if !ok {
|
||||
slog.Warn("security.mcp_bridge: invalid bridge context signature",
|
||||
"agent_id", agentIDStr, "user_id", userID)
|
||||
@@ -448,6 +487,10 @@ func bridgeContextMiddleware(gatewayToken string, agentStore store.AgentStore, n
|
||||
if workspace != "" && (agentIDStr != "" || userID != "") {
|
||||
ctx = tools.WithToolWorkspace(ctx, workspace)
|
||||
}
|
||||
if delegationID != "" && delegationInputs != "" && (agentIDStr != "" || userID != "") {
|
||||
ctx = tools.WithDelegationID(ctx, delegationID)
|
||||
ctx = tools.WithDelegationArtifactInputs(ctx, delegationInputs)
|
||||
}
|
||||
// Routing context (localKey, sessionKey) is injected unconditionally like channel/chatID.
|
||||
// These are used for message routing (forum topics), not security-sensitive operations.
|
||||
// Without valid agent context, tool execution will fail anyway.
|
||||
@@ -514,9 +557,9 @@ func (s *Server) Start(ctx context.Context) error {
|
||||
s.httpServer = &http.Server{
|
||||
Addr: addr,
|
||||
Handler: handler,
|
||||
ReadTimeout: 3600 * time.Second, // 1h: allow large uploads, long-running reads
|
||||
WriteTimeout: 3600 * time.Second, // 1h: allow streaming responses, slow clients
|
||||
IdleTimeout: 30 * time.Second, // 30s: close idle connections
|
||||
ReadTimeout: 3600 * time.Second, // 1h: allow large uploads, long-running reads
|
||||
WriteTimeout: 3600 * time.Second, // 1h: allow streaming responses, slow clients
|
||||
IdleTimeout: 30 * time.Second, // 30s: close idle connections
|
||||
}
|
||||
|
||||
slog.Info("gateway starting", "addr", addr)
|
||||
|
||||
@@ -107,5 +107,5 @@ func EmitHookSpan(
|
||||
if collector == nil {
|
||||
return // tracing disabled — no collector attached
|
||||
}
|
||||
collector.EmitSpan(span)
|
||||
collector.EmitSpan(tracing.RedactSpan(ctx, span))
|
||||
}
|
||||
@@ -205,6 +205,10 @@ func makeToolHandler(reg *tools.Registry, toolName string, msgBus *bus.MessageBu
|
||||
return mcpgo.NewToolResultError(result.ForLLM), nil
|
||||
}
|
||||
|
||||
// A delegated artifact is not outbound media until DelegateTool validates
|
||||
// and atomically publishes its manifest back into the caller workspace.
|
||||
tools.ApplyDelegationArtifactResultPolicy(ctx, result)
|
||||
|
||||
// Forward media files to the outbound bus so they reach the user as attachments.
|
||||
// This is necessary because Claude CLI processes tool results internally —
|
||||
// GoClaw's agent loop never sees result.Media from bridge tool calls.
|
||||
@@ -216,7 +220,7 @@ func makeToolHandler(reg *tools.Registry, toolName string, msgBus *bus.MessageBu
|
||||
|
||||
// forwardMediaToOutbound publishes media files from a tool result to the outbound bus.
|
||||
func forwardMediaToOutbound(ctx context.Context, msgBus *bus.MessageBus, toolName string, result *tools.Result) {
|
||||
if msgBus == nil || len(result.Media) == 0 {
|
||||
if msgBus == nil || tools.IsDelegationArtifactRun(ctx) || len(result.Media) == 0 {
|
||||
return
|
||||
}
|
||||
channel := tools.ToolChannelFromCtx(ctx)
|
||||
|
||||
@@ -3,9 +3,11 @@ package mcp
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mcpgo "github.com/mark3labs/mcp-go/mcp"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/bus"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/config"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/tools"
|
||||
)
|
||||
@@ -141,3 +143,21 @@ func toolNames(ts []mcpgo.Tool) []string {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestForwardMediaToOutboundSkipsDelegationArtifactRun(t *testing.T) {
|
||||
msgBus := bus.New()
|
||||
ctx := tools.WithToolChannel(context.Background(), "telegram")
|
||||
ctx = tools.WithToolChatID(ctx, "chat-id")
|
||||
ctx = tools.WithDelegationID(ctx, "delegation-id")
|
||||
ctx = tools.WithDelegationArtifactInputs(ctx, "/runtime/inputs")
|
||||
|
||||
forwardMediaToOutbound(ctx, msgBus, "create_image", &tools.Result{
|
||||
Media: []bus.MediaFile{{Path: "/runtime/outputs/generated.png", MimeType: "image/png"}},
|
||||
})
|
||||
|
||||
waitCtx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
|
||||
defer cancel()
|
||||
if msg, ok := msgBus.SubscribeOutbound(waitCtx); ok {
|
||||
t.Fatalf("unpublished delegation media forwarded: %#v", msg)
|
||||
}
|
||||
}
|
||||
@@ -28,7 +28,7 @@ func registerAgentLinkCRUDTools(srv *mcpserver.MCPServer, links store.AgentLinkS
|
||||
mcpgo.WithString("target_agent", mcpgo.Required(), mcpgo.Description("Target agent UUID.")),
|
||||
mcpgo.WithString("direction", mcpgo.Enum("outbound", "inbound", "bidirectional"), mcpgo.Description("Link direction; defaults to \"outbound\".")),
|
||||
mcpgo.WithString("description", mcpgo.Description("Human-readable description of the link.")),
|
||||
mcpgo.WithNumber("max_concurrent", mcpgo.Description("Maximum concurrent delegated tasks.")),
|
||||
mcpgo.WithNumber("max_concurrent", mcpgo.Description("Compatibility metadata reserved for future per-link scheduling; not enforced at runtime.")),
|
||||
), handleAgentLinksCreate(links))
|
||||
|
||||
srv.AddTool(mcpgo.NewTool("goclaw_agent_links_update",
|
||||
@@ -36,7 +36,7 @@ func registerAgentLinkCRUDTools(srv *mcpserver.MCPServer, links store.AgentLinkS
|
||||
mcpgo.WithString("link_id", mcpgo.Required(), mcpgo.Description("Link UUID.")),
|
||||
mcpgo.WithString("direction", mcpgo.Enum("outbound", "inbound", "bidirectional"), mcpgo.Description("New direction.")),
|
||||
mcpgo.WithString("description", mcpgo.Description("New description.")),
|
||||
mcpgo.WithNumber("max_concurrent", mcpgo.Description("New max concurrent delegated tasks.")),
|
||||
mcpgo.WithNumber("max_concurrent", mcpgo.Description("New compatibility metadata value reserved for future per-link scheduling; not enforced at runtime.")),
|
||||
mcpgo.WithString("status", mcpgo.Enum("active", "disabled"), mcpgo.Description("New status.")),
|
||||
), handleAgentLinksUpdate(links))
|
||||
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
package mcp
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
)
|
||||
|
||||
func TestAgentLinkMaxConcurrentSchemaIsCompatibilityMetadata(t *testing.T) {
|
||||
srv := newTestMCPServer()
|
||||
var links store.AgentLinkStore
|
||||
registerAgentLinkCRUDTools(srv, links)
|
||||
|
||||
for _, toolName := range []string{
|
||||
"goclaw_agent_links_create",
|
||||
"goclaw_agent_links_update",
|
||||
} {
|
||||
registered := srv.GetTool(toolName)
|
||||
if registered == nil {
|
||||
t.Fatalf("%s was not registered", toolName)
|
||||
}
|
||||
property, ok := registered.Tool.InputSchema.Properties["max_concurrent"]
|
||||
if !ok {
|
||||
t.Fatalf("%s omitted max_concurrent compatibility field", toolName)
|
||||
}
|
||||
encoded, err := json.Marshal(property)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal %s max_concurrent schema: %v", toolName, err)
|
||||
}
|
||||
description := strings.ToLower(string(encoded))
|
||||
if !strings.Contains(description, "compatibility metadata") ||
|
||||
!strings.Contains(description, "not enforced") {
|
||||
t.Fatalf("%s exposes a misleading max_concurrent contract: %s", toolName, encoded)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -70,6 +70,12 @@ func (s *Store) LoadPath(id string) (string, error) {
|
||||
return matches[0], nil
|
||||
}
|
||||
|
||||
// MediaRootPath exposes the managed legacy-media boundary to media reader
|
||||
// tools. Returned paths are still containment-checked by the caller.
|
||||
func (s *Store) MediaRootPath() string {
|
||||
return s.baseDir
|
||||
}
|
||||
|
||||
// DeleteSession removes all media files for a session.
|
||||
func (s *Store) DeleteSession(sessionKey string) error {
|
||||
dir := s.sessionDir(sessionKey)
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
package orchestration
|
||||
|
||||
import "github.com/nextlevelbuilder/goclaw/internal/childrun"
|
||||
|
||||
var (
|
||||
ErrChildRunBusy = childrun.ErrChildRunBusy
|
||||
ErrChildRunClosed = childrun.ErrChildRunClosed
|
||||
ErrChildRunDrainTimeout = childrun.ErrChildRunDrainTimeout
|
||||
ErrChildRunInvalidContinuation = childrun.ErrChildRunInvalidContinuation
|
||||
ErrChildRunContinuationDepth = childrun.ErrChildRunContinuationDepth
|
||||
)
|
||||
|
||||
type ChildRunState = childrun.ChildRunState
|
||||
|
||||
const (
|
||||
ChildRunNew = childrun.ChildRunNew
|
||||
ChildRunQueued = childrun.ChildRunQueued
|
||||
ChildRunRunning = childrun.ChildRunRunning
|
||||
ChildRunWaitingChild = childrun.ChildRunWaitingChild
|
||||
ChildRunCompleted = childrun.ChildRunCompleted
|
||||
ChildRunFailed = childrun.ChildRunFailed
|
||||
ChildRunCancelled = childrun.ChildRunCancelled
|
||||
)
|
||||
|
||||
type ChildRunConstraints = childrun.ChildRunConstraints
|
||||
type ChildRunStats = childrun.ChildRunStats
|
||||
type ChildRunAdmission = childrun.ChildRunAdmission
|
||||
type ChildRunTicket = childrun.ChildRunTicket
|
||||
type ChildRunLease = childrun.ChildRunLease
|
||||
|
||||
func NewChildRunAdmission(processLimit, pendingLimit int) *ChildRunAdmission {
|
||||
return childrun.NewChildRunAdmission(processLimit, pendingLimit)
|
||||
}
|
||||
@@ -32,6 +32,11 @@ const OptPeerKind = "peer_kind"
|
||||
// OptWorkspace passes the agent workspace path so MCP bridge tools can resolve file paths.
|
||||
const OptWorkspace = "workspace"
|
||||
|
||||
// OptDelegationID and OptDelegationInputs pass signed artifact-run context to
|
||||
// MCP bridge tools. They are runtime-only and are not included in model prompts.
|
||||
const OptDelegationID = "delegation_id"
|
||||
const OptDelegationInputs = "delegation_inputs"
|
||||
|
||||
// OptTenantID passes the tenant UUID string for per-session MCP config.
|
||||
// Required for memory indexing and tenant-scoped queries via bridge tools.
|
||||
const OptTenantID = "tenant_id"
|
||||
|
||||
@@ -84,14 +84,16 @@ func mcpConfigBaseDir() string {
|
||||
|
||||
// BridgeContext holds per-call context for MCP bridge headers.
|
||||
type BridgeContext struct {
|
||||
AgentID string
|
||||
UserID string
|
||||
Channel string
|
||||
ChatID string
|
||||
PeerKind string
|
||||
Workspace string
|
||||
TenantID string
|
||||
LocalKey string
|
||||
AgentID string
|
||||
UserID string
|
||||
Channel string
|
||||
ChatID string
|
||||
PeerKind string
|
||||
Workspace string
|
||||
TenantID string
|
||||
LocalKey string
|
||||
DelegationID string
|
||||
DelegationInputs string
|
||||
}
|
||||
|
||||
// WriteMCPConfig writes a per-session MCP config file with agent context headers.
|
||||
@@ -99,10 +101,27 @@ type BridgeContext struct {
|
||||
// outside the agent's workDir so tokens are not exposed.
|
||||
// Skips write if content is unchanged. Returns the file path.
|
||||
func (d *MCPConfigData) WriteMCPConfig(ctx context.Context, sessionKey string, bc BridgeContext) string {
|
||||
return d.writeMCPConfigInternal(ctx, sessionKey, bc.AgentID, bc.UserID, bc.Channel, bc.ChatID, bc.PeerKind, bc.Workspace, bc.TenantID, bc.LocalKey)
|
||||
return d.writeMCPConfigInternal(
|
||||
ctx,
|
||||
sessionKey,
|
||||
bc.AgentID,
|
||||
bc.UserID,
|
||||
bc.Channel,
|
||||
bc.ChatID,
|
||||
bc.PeerKind,
|
||||
bc.Workspace,
|
||||
bc.TenantID,
|
||||
bc.LocalKey,
|
||||
bc.DelegationID,
|
||||
bc.DelegationInputs,
|
||||
)
|
||||
}
|
||||
|
||||
func (d *MCPConfigData) writeMCPConfigInternal(ctx context.Context, sessionKey, agentID, userID, channel, chatID, peerKind, workspace, tenantID, localKey string) string {
|
||||
func (d *MCPConfigData) writeMCPConfigInternal(
|
||||
ctx context.Context,
|
||||
sessionKey, agentID, userID, channel, chatID, peerKind, workspace, tenantID, localKey string,
|
||||
delegationID, delegationInputs string,
|
||||
) string {
|
||||
if d == nil || (len(d.Servers) == 0 && d.GatewayAddr == "" && d.AgentMCPLookup == nil) {
|
||||
return ""
|
||||
}
|
||||
@@ -158,9 +177,29 @@ func (d *MCPConfigData) writeMCPConfigInternal(ctx context.Context, sessionKey,
|
||||
if sessionKey != "" && !strings.ContainsAny(sessionKey, "\r\n\x00") {
|
||||
headers["X-Session-Key"] = sessionKey
|
||||
}
|
||||
if delegationID != "" && !strings.ContainsAny(delegationID, "\r\n\x00") {
|
||||
headers["X-Delegation-ID"] = delegationID
|
||||
}
|
||||
if delegationInputs != "" && !strings.ContainsAny(delegationInputs, "\r\n\x00") {
|
||||
headers["X-Delegation-Inputs"] = delegationInputs
|
||||
}
|
||||
// HMAC signature over all context fields to prevent header forgery
|
||||
if d.GatewayToken != "" && (agentID != "" || userID != "") {
|
||||
headers["X-Bridge-Sig"] = SignBridgeContext(d.GatewayToken, agentID, userID, channel, chatID, peerKind, workspace, tenantID, localKey, sessionKey)
|
||||
extra := []string{localKey, sessionKey}
|
||||
if delegationID != "" || delegationInputs != "" {
|
||||
extra = append(extra, delegationID, delegationInputs)
|
||||
}
|
||||
headers["X-Bridge-Sig"] = SignBridgeContext(
|
||||
d.GatewayToken,
|
||||
agentID,
|
||||
userID,
|
||||
channel,
|
||||
chatID,
|
||||
peerKind,
|
||||
workspace,
|
||||
tenantID,
|
||||
extra...,
|
||||
)
|
||||
}
|
||||
|
||||
bridgeEntry := map[string]any{
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package providers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -179,3 +182,66 @@ func TestVerifyBridgeContext_ExtraParamOrderMatters(t *testing.T) {
|
||||
t.Error("expected ok=false for swapped extra param order")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBridgeContextFromOptsCarriesDelegationArtifactContext(t *testing.T) {
|
||||
got := bridgeContextFromOpts(map[string]any{
|
||||
OptAgentID: "agent-id",
|
||||
OptWorkspace: "/runtime/outputs",
|
||||
OptDelegationID: "delegation-id",
|
||||
OptDelegationInputs: "/runtime/inputs",
|
||||
})
|
||||
|
||||
if got.DelegationID != "delegation-id" || got.DelegationInputs != "/runtime/inputs" {
|
||||
t.Fatalf("delegation bridge context = %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteMCPConfigSignsDelegationArtifactHeaders(t *testing.T) {
|
||||
t.Setenv("GOCLAW_DATA_DIR", t.TempDir())
|
||||
const (
|
||||
token = "gateway-token"
|
||||
sessionKey = "session-key"
|
||||
)
|
||||
data := &MCPConfigData{GatewayAddr: "127.0.0.1:18790", GatewayToken: token}
|
||||
bc := BridgeContext{
|
||||
AgentID: "agent-id",
|
||||
Workspace: "/runtime/outputs",
|
||||
DelegationID: "delegation-id",
|
||||
DelegationInputs: "/runtime/inputs",
|
||||
}
|
||||
path := data.WriteMCPConfig(context.Background(), sessionKey, bc)
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var cfg struct {
|
||||
MCPServers map[string]struct {
|
||||
Headers map[string]string `json:"headers"`
|
||||
} `json:"mcpServers"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
headers := cfg.MCPServers["goclaw-bridge"].Headers
|
||||
if headers["X-Delegation-ID"] != bc.DelegationID ||
|
||||
headers["X-Delegation-Inputs"] != bc.DelegationInputs {
|
||||
t.Fatalf("delegation headers = %#v", headers)
|
||||
}
|
||||
wantSig := SignBridgeContext(
|
||||
token,
|
||||
bc.AgentID,
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
bc.Workspace,
|
||||
"",
|
||||
"",
|
||||
sessionKey,
|
||||
bc.DelegationID,
|
||||
bc.DelegationInputs,
|
||||
)
|
||||
if headers["X-Bridge-Sig"] != wantSig {
|
||||
t.Fatal("delegation headers were not covered by the exact bridge signature")
|
||||
}
|
||||
}
|
||||
@@ -285,14 +285,16 @@ func extractStringSliceOpt(opts map[string]any, key string) []string {
|
||||
// bridgeContextFromOpts builds a BridgeContext from the Options map.
|
||||
func bridgeContextFromOpts(opts map[string]any) BridgeContext {
|
||||
return BridgeContext{
|
||||
AgentID: extractStringOpt(opts, OptAgentID),
|
||||
UserID: extractStringOpt(opts, OptUserID),
|
||||
Channel: extractStringOpt(opts, OptChannel),
|
||||
ChatID: extractStringOpt(opts, OptChatID),
|
||||
PeerKind: extractStringOpt(opts, OptPeerKind),
|
||||
Workspace: extractStringOpt(opts, OptWorkspace),
|
||||
TenantID: extractStringOpt(opts, OptTenantID),
|
||||
LocalKey: extractStringOpt(opts, OptLocalKey),
|
||||
AgentID: extractStringOpt(opts, OptAgentID),
|
||||
UserID: extractStringOpt(opts, OptUserID),
|
||||
Channel: extractStringOpt(opts, OptChannel),
|
||||
ChatID: extractStringOpt(opts, OptChatID),
|
||||
PeerKind: extractStringOpt(opts, OptPeerKind),
|
||||
Workspace: extractStringOpt(opts, OptWorkspace),
|
||||
TenantID: extractStringOpt(opts, OptTenantID),
|
||||
LocalKey: extractStringOpt(opts, OptLocalKey),
|
||||
DelegationID: extractStringOpt(opts, OptDelegationID),
|
||||
DelegationInputs: extractStringOpt(opts, OptDelegationInputs),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+57
-42
@@ -3,8 +3,6 @@ package sandbox
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"maps"
|
||||
@@ -34,12 +32,19 @@ type DockerSandbox struct {
|
||||
workspace string
|
||||
createdAt time.Time
|
||||
lastUsed time.Time
|
||||
destroyFunc func(context.Context) error
|
||||
mu sync.Mutex // protects lastUsed
|
||||
}
|
||||
|
||||
// newDockerSandbox creates and starts a Docker container for sandboxed execution.
|
||||
// Matching TS buildSandboxCreateArgs() + createSandboxContainer().
|
||||
func newDockerSandbox(ctx context.Context, name string, cfg Config, workspace string) (*DockerSandbox, error) {
|
||||
func newDockerSandbox(
|
||||
ctx context.Context,
|
||||
name string,
|
||||
cfg Config,
|
||||
workspace string,
|
||||
readOnlyMounts ...ReadOnlyMount,
|
||||
) (*DockerSandbox, error) {
|
||||
args := []string{
|
||||
"run", "-d",
|
||||
"--name", name,
|
||||
@@ -90,17 +95,18 @@ func newDockerSandbox(ctx context.Context, name string, cfg Config, workspace st
|
||||
args = append(args, "--network", "none")
|
||||
}
|
||||
|
||||
// Workspace mount — resolve host path for DooD (Docker-out-of-Docker) setups.
|
||||
containerWorkdir := cfg.ContainerWorkdir()
|
||||
if workspace != "" && cfg.WorkspaceAccess != AccessNone {
|
||||
mountOpt := "rw"
|
||||
if cfg.WorkspaceAccess == AccessRO {
|
||||
mountOpt = "ro"
|
||||
}
|
||||
hostPath := resolveHostWorkspacePath(ctx, workspace)
|
||||
args = append(args, "-v", fmt.Sprintf("%s:%s:%s", hostPath, containerWorkdir, mountOpt))
|
||||
args, sensitiveMountRoots, err := appendDockerMountArgs(
|
||||
ctx,
|
||||
args,
|
||||
cfg,
|
||||
workspace,
|
||||
readOnlyMounts,
|
||||
resolveHostWorkspacePath,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
args = append(args, "-w", containerWorkdir)
|
||||
args = append(args, "-w", cfg.ContainerWorkdir())
|
||||
|
||||
// Environment variables
|
||||
for k, v := range cfg.Env {
|
||||
@@ -110,7 +116,12 @@ func newDockerSandbox(ctx context.Context, name string, cfg Config, workspace st
|
||||
// Image + keep-alive command
|
||||
args = append(args, cfg.Image, "sleep", "infinity")
|
||||
|
||||
slog.Debug("creating sandbox container", "name", name, "args", args)
|
||||
slog.Debug(
|
||||
"creating sandbox container",
|
||||
"name", name,
|
||||
"image", cfg.Image,
|
||||
"additional_read_only_mounts", len(readOnlyMounts),
|
||||
)
|
||||
|
||||
cmd := exec.CommandContext(ctx, "docker", args...)
|
||||
var stdout, stderr bytes.Buffer
|
||||
@@ -118,7 +129,8 @@ func newDockerSandbox(ctx context.Context, name string, cfg Config, workspace st
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
return nil, fmt.Errorf("docker run failed: %w\nstderr: %s", err, stderr.String())
|
||||
safeStderr := redactMountRoots(stderr.String(), sensitiveMountRoots)
|
||||
return nil, fmt.Errorf("docker run failed: %w\nstderr: %s", err, safeStderr)
|
||||
}
|
||||
|
||||
containerID := strings.TrimSpace(stdout.String())
|
||||
@@ -214,6 +226,9 @@ func (s *DockerSandbox) Exec(ctx context.Context, command []string, workDir stri
|
||||
|
||||
// Destroy removes the container.
|
||||
func (s *DockerSandbox) Destroy(ctx context.Context) error {
|
||||
if s.destroyFunc != nil {
|
||||
return s.destroyFunc(ctx)
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "docker", "rm", "-f", s.containerID)
|
||||
if err := cmd.Run(); err != nil {
|
||||
slog.Warn("failed to remove sandbox container", "id", s.containerID, "error", err)
|
||||
@@ -234,6 +249,8 @@ type DockerManager struct {
|
||||
stopCh chan struct{} // signals pruning goroutine to stop
|
||||
}
|
||||
|
||||
var _ Manager = (*DockerManager)(nil)
|
||||
|
||||
// NewDockerManager creates a manager for Docker sandboxes.
|
||||
// Automatically starts background pruning if configured.
|
||||
func NewDockerManager(cfg Config) *DockerManager {
|
||||
@@ -248,7 +265,13 @@ func NewDockerManager(cfg Config) *DockerManager {
|
||||
|
||||
// Get returns an existing sandbox or creates a new one for the given key.
|
||||
// If cfgOverride is non-nil, it is used for new containers instead of the global config.
|
||||
func (m *DockerManager) Get(ctx context.Context, key string, workspace string, cfgOverride *Config) (Sandbox, error) {
|
||||
func (m *DockerManager) Get(
|
||||
ctx context.Context,
|
||||
key string,
|
||||
workspace string,
|
||||
cfgOverride *Config,
|
||||
opts ...GetOption,
|
||||
) (Sandbox, error) {
|
||||
cfg := m.config
|
||||
if cfgOverride != nil {
|
||||
cfg = *cfgOverride
|
||||
@@ -256,7 +279,13 @@ func (m *DockerManager) Get(ctx context.Context, key string, workspace string, c
|
||||
if cfg.Mode == ModeOff {
|
||||
return nil, ErrSandboxDisabled
|
||||
}
|
||||
cacheKey := dockerCacheKey(key, workspace, cfg)
|
||||
getOpts := ApplyGetOpts(opts)
|
||||
cfg = applyGetOptsToConfig(cfg, getOpts)
|
||||
readOnlyMounts, err := validateReadOnlyMounts(cfg, getOpts.ReadOnlyMounts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cacheKey := dockerCacheKey(key, workspace, cfg, readOnlyMounts...)
|
||||
|
||||
m.mu.RLock()
|
||||
if sb, ok := m.sandboxes[cacheKey]; ok {
|
||||
@@ -278,7 +307,7 @@ func (m *DockerManager) Get(ctx context.Context, key string, workspace string, c
|
||||
prefix = "goclaw-sbx-"
|
||||
}
|
||||
name := prefix + sanitizeKey(cacheKey)
|
||||
sb, err := newDockerSandbox(ctx, name, cfg, workspace)
|
||||
sb, err := newDockerSandbox(ctx, name, cfg, workspace, readOnlyMounts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -287,35 +316,21 @@ func (m *DockerManager) Get(ctx context.Context, key string, workspace string, c
|
||||
return sb, nil
|
||||
}
|
||||
|
||||
func dockerCacheKey(key, workspace string, cfg Config) string {
|
||||
if workspace == "" {
|
||||
return key
|
||||
}
|
||||
h := sha256.Sum256([]byte(strings.Join([]string{
|
||||
workspace,
|
||||
string(cfg.WorkspaceAccess),
|
||||
cfg.ContainerWorkdir(),
|
||||
cfg.Image,
|
||||
}, "\x00")))
|
||||
return "w" + hex.EncodeToString(h[:])[:16] + ":" + key
|
||||
}
|
||||
|
||||
// Release destroys a sandbox by key.
|
||||
func (m *DockerManager) Release(ctx context.Context, key string) error {
|
||||
m.mu.Lock()
|
||||
sbs := make([]*DockerSandbox, 0, 1)
|
||||
for cacheKey, sb := range m.sandboxes {
|
||||
if cacheKey == key || strings.HasSuffix(cacheKey, ":"+key) {
|
||||
delete(m.sandboxes, cacheKey)
|
||||
sbs = append(sbs, sb)
|
||||
}
|
||||
}
|
||||
m.mu.Unlock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
var firstErr error
|
||||
for _, sb := range sbs {
|
||||
if err := sb.Destroy(ctx); err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
for cacheKey, sb := range m.sandboxes {
|
||||
if cacheKey == key || strings.HasSuffix(cacheKey, ":"+key) {
|
||||
if err := sb.Destroy(ctx); err != nil {
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
continue
|
||||
}
|
||||
delete(m.sandboxes, cacheKey)
|
||||
}
|
||||
}
|
||||
return firstErr
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package sandbox
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDockerCacheKeyIncludesWorkspaceAndConfig(t *testing.T) {
|
||||
cfg := DefaultConfig()
|
||||
@@ -28,3 +31,37 @@ func TestDockerCacheKeyPreservesEmptyWorkspaceCompatibility(t *testing.T) {
|
||||
t.Fatalf("dockerCacheKey empty workspace = %q, want original key %q", got, key)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDockerCacheKeyDoesNotExposeMountRoots(t *testing.T) {
|
||||
cfg := DefaultConfig()
|
||||
hostRoot := "/sensitive/runtime/delegation/inputs"
|
||||
got := dockerCacheKey(
|
||||
"delegation:9db9",
|
||||
"/sensitive/runtime/delegation/outputs",
|
||||
cfg,
|
||||
ReadOnlyMount{
|
||||
Name: "inputs",
|
||||
HostPath: hostRoot,
|
||||
Destination: "/workspace/inputs",
|
||||
},
|
||||
)
|
||||
|
||||
if strings.Contains(got, "/sensitive/") || strings.Contains(got, hostRoot) {
|
||||
t.Fatalf("dockerCacheKey exposed mount root: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDockerCacheKeyIncludesDelegationIdentity(t *testing.T) {
|
||||
cfg := DefaultConfig()
|
||||
mount := ReadOnlyMount{
|
||||
Name: "inputs",
|
||||
HostPath: "/runtime/inputs",
|
||||
Destination: "/workspace/inputs",
|
||||
}
|
||||
|
||||
first := dockerCacheKey("delegation:first", "/runtime/outputs", cfg, mount)
|
||||
second := dockerCacheKey("delegation:second", "/runtime/outputs", cfg, mount)
|
||||
if first == second {
|
||||
t.Fatalf("dockerCacheKey reused identity across delegation keys: %q", first)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
package sandbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const redactedMountRoot = "[REDACTED_MOUNT_ROOT]"
|
||||
|
||||
type mountPathResolver func(context.Context, string) string
|
||||
|
||||
func validateReadOnlyMounts(cfg Config, mounts []ReadOnlyMount) ([]ReadOnlyMount, error) {
|
||||
if len(mounts) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if cfg.WorkspaceAccess == AccessNone {
|
||||
return nil, fmt.Errorf("read-only mounts require sandbox workspace access")
|
||||
}
|
||||
|
||||
workdir := cfg.ContainerWorkdir()
|
||||
if !path.IsAbs(workdir) || path.Clean(workdir) != workdir {
|
||||
return nil, fmt.Errorf("sandbox workdir must be an absolute canonical path")
|
||||
}
|
||||
|
||||
normalized := append([]ReadOnlyMount(nil), mounts...)
|
||||
names := make(map[string]struct{}, len(normalized))
|
||||
destinations := make(map[string]struct{}, len(normalized))
|
||||
|
||||
for i := range normalized {
|
||||
mount := &normalized[i]
|
||||
if mount.Name == "" || strings.TrimSpace(mount.Name) != mount.Name || strings.ContainsRune(mount.Name, '\x00') {
|
||||
return nil, fmt.Errorf("read-only mount %d has an invalid name", i)
|
||||
}
|
||||
if _, exists := names[mount.Name]; exists {
|
||||
return nil, fmt.Errorf("read-only mounts must have unique names")
|
||||
}
|
||||
names[mount.Name] = struct{}{}
|
||||
|
||||
if !filepath.IsAbs(mount.HostPath) || filepath.Clean(mount.HostPath) != mount.HostPath {
|
||||
return nil, fmt.Errorf("read-only mount %d host path must be absolute and canonical", i)
|
||||
}
|
||||
canonicalHost, err := filepath.EvalSymlinks(mount.HostPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read-only mount %d host path cannot be resolved", i)
|
||||
}
|
||||
if canonicalHost != mount.HostPath {
|
||||
return nil, fmt.Errorf("read-only mount %d host path must be absolute and canonical", i)
|
||||
}
|
||||
info, err := os.Stat(mount.HostPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read-only mount %d host path cannot be inspected", i)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return nil, fmt.Errorf("read-only mount %d host path must be a directory", i)
|
||||
}
|
||||
|
||||
if strings.ContainsRune(mount.Destination, '\x00') ||
|
||||
!path.IsAbs(mount.Destination) ||
|
||||
path.Clean(mount.Destination) != mount.Destination {
|
||||
return nil, fmt.Errorf("read-only mount %d destination must be absolute and canonical", i)
|
||||
}
|
||||
if !containerPathStrictlyWithin(workdir, mount.Destination) {
|
||||
return nil, fmt.Errorf("read-only mount %d destination must be strictly beneath sandbox workdir", i)
|
||||
}
|
||||
if _, exists := destinations[mount.Destination]; exists {
|
||||
return nil, fmt.Errorf("read-only mounts must have unique destinations")
|
||||
}
|
||||
destinations[mount.Destination] = struct{}{}
|
||||
}
|
||||
|
||||
sort.Slice(normalized, func(i, j int) bool {
|
||||
if normalized[i].Name != normalized[j].Name {
|
||||
return normalized[i].Name < normalized[j].Name
|
||||
}
|
||||
if normalized[i].Destination != normalized[j].Destination {
|
||||
return normalized[i].Destination < normalized[j].Destination
|
||||
}
|
||||
return normalized[i].HostPath < normalized[j].HostPath
|
||||
})
|
||||
return normalized, nil
|
||||
}
|
||||
|
||||
func containerPathStrictlyWithin(root, target string) bool {
|
||||
if target == root {
|
||||
return false
|
||||
}
|
||||
return strings.HasPrefix(target, strings.TrimSuffix(root, "/")+"/")
|
||||
}
|
||||
|
||||
func appendDockerMountArgs(
|
||||
ctx context.Context,
|
||||
args []string,
|
||||
cfg Config,
|
||||
workspace string,
|
||||
mounts []ReadOnlyMount,
|
||||
resolve mountPathResolver,
|
||||
) ([]string, []string, error) {
|
||||
normalized, err := validateReadOnlyMounts(cfg, mounts)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if resolve == nil {
|
||||
resolve = func(_ context.Context, path string) string { return path }
|
||||
}
|
||||
|
||||
sensitiveRoots := make([]string, 0, 2+len(normalized)*2)
|
||||
containerWorkdir := cfg.ContainerWorkdir()
|
||||
if workspace != "" && cfg.WorkspaceAccess != AccessNone {
|
||||
mountOpt := "rw"
|
||||
if cfg.WorkspaceAccess == AccessRO {
|
||||
mountOpt = "ro"
|
||||
}
|
||||
hostPath := resolve(ctx, workspace)
|
||||
sensitiveRoots = appendSensitiveRoot(sensitiveRoots, workspace)
|
||||
sensitiveRoots = appendSensitiveRoot(sensitiveRoots, hostPath)
|
||||
args = append(args, "-v", fmt.Sprintf("%s:%s:%s", hostPath, containerWorkdir, mountOpt))
|
||||
}
|
||||
|
||||
for _, mount := range normalized {
|
||||
hostPath := resolve(ctx, mount.HostPath)
|
||||
sensitiveRoots = appendSensitiveRoot(sensitiveRoots, mount.HostPath)
|
||||
sensitiveRoots = appendSensitiveRoot(sensitiveRoots, hostPath)
|
||||
args = append(args, "-v", fmt.Sprintf("%s:%s:ro", hostPath, mount.Destination))
|
||||
}
|
||||
return args, sensitiveRoots, nil
|
||||
}
|
||||
|
||||
func appendSensitiveRoot(roots []string, root string) []string {
|
||||
if root == "" {
|
||||
return roots
|
||||
}
|
||||
if slices.Contains(roots, root) {
|
||||
return roots
|
||||
}
|
||||
return append(roots, root)
|
||||
}
|
||||
|
||||
func redactMountRoots(message string, roots []string) string {
|
||||
sorted := append([]string(nil), roots...)
|
||||
sort.Slice(sorted, func(i, j int) bool {
|
||||
return len(sorted[i]) > len(sorted[j])
|
||||
})
|
||||
for _, root := range sorted {
|
||||
if root != "" {
|
||||
message = strings.ReplaceAll(message, root, redactedMountRoot)
|
||||
}
|
||||
}
|
||||
return message
|
||||
}
|
||||
|
||||
func dockerCacheKey(key, workspace string, cfg Config, mounts ...ReadOnlyMount) string {
|
||||
if workspace == "" && len(mounts) == 0 {
|
||||
return key
|
||||
}
|
||||
|
||||
orderedMounts := append([]ReadOnlyMount(nil), mounts...)
|
||||
sort.Slice(orderedMounts, func(i, j int) bool {
|
||||
if orderedMounts[i].Name != orderedMounts[j].Name {
|
||||
return orderedMounts[i].Name < orderedMounts[j].Name
|
||||
}
|
||||
if orderedMounts[i].Destination != orderedMounts[j].Destination {
|
||||
return orderedMounts[i].Destination < orderedMounts[j].Destination
|
||||
}
|
||||
return orderedMounts[i].HostPath < orderedMounts[j].HostPath
|
||||
})
|
||||
|
||||
parts := []string{
|
||||
workspace,
|
||||
string(cfg.WorkspaceAccess),
|
||||
cfg.ContainerWorkdir(),
|
||||
cfg.Image,
|
||||
}
|
||||
for _, mount := range orderedMounts {
|
||||
parts = append(parts, mount.Name, mount.HostPath, mount.Destination, "ro")
|
||||
}
|
||||
h := sha256.Sum256([]byte(strings.Join(parts, "\x00")))
|
||||
return "w" + hex.EncodeToString(h[:])[:16] + ":" + key
|
||||
}
|
||||
@@ -0,0 +1,342 @@
|
||||
package sandbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestValidateReadOnlyMountsCanonicalizesOrder(t *testing.T) {
|
||||
first := canonicalTempDir(t)
|
||||
second := canonicalTempDir(t)
|
||||
cfg := DefaultConfig()
|
||||
|
||||
got, err := validateReadOnlyMounts(cfg, []ReadOnlyMount{
|
||||
{Name: "zeta", HostPath: second, Destination: "/workspace/zeta"},
|
||||
{Name: "alpha", HostPath: first, Destination: "/workspace/alpha"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("validateReadOnlyMounts() error = %v", err)
|
||||
}
|
||||
if got[0].Name != "alpha" || got[1].Name != "zeta" {
|
||||
t.Fatalf("validateReadOnlyMounts() order = %#v, want name-sorted mounts", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateReadOnlyMountsRejectsInvalidContractsWithoutLeakingRoots(t *testing.T) {
|
||||
host := canonicalTempDir(t)
|
||||
otherHost := canonicalTempDir(t)
|
||||
file := filepath.Join(host, "file.txt")
|
||||
if err := os.WriteFile(file, []byte("test"), 0o600); err != nil {
|
||||
t.Fatalf("write fixture: %v", err)
|
||||
}
|
||||
symlink := filepath.Join(otherHost, "linked")
|
||||
if err := os.Symlink(host, symlink); err != nil {
|
||||
t.Fatalf("create symlink fixture: %v", err)
|
||||
}
|
||||
|
||||
cfg := DefaultConfig()
|
||||
tests := []struct {
|
||||
name string
|
||||
cfg Config
|
||||
mounts []ReadOnlyMount
|
||||
}{
|
||||
{
|
||||
name: "empty name",
|
||||
cfg: cfg,
|
||||
mounts: []ReadOnlyMount{{HostPath: host, Destination: "/workspace/inputs"}},
|
||||
},
|
||||
{
|
||||
name: "duplicate name",
|
||||
cfg: cfg,
|
||||
mounts: []ReadOnlyMount{{Name: "inputs", HostPath: host, Destination: "/workspace/a"}, {Name: "inputs", HostPath: otherHost, Destination: "/workspace/b"}},
|
||||
},
|
||||
{
|
||||
name: "relative host",
|
||||
cfg: cfg,
|
||||
mounts: []ReadOnlyMount{{Name: "inputs", HostPath: "relative", Destination: "/workspace/inputs"}},
|
||||
},
|
||||
{
|
||||
name: "unclean host",
|
||||
cfg: cfg,
|
||||
mounts: []ReadOnlyMount{{Name: "inputs", HostPath: host + string(filepath.Separator) + ".." + string(filepath.Separator) + filepath.Base(host), Destination: "/workspace/inputs"}},
|
||||
},
|
||||
{
|
||||
name: "symlink host",
|
||||
cfg: cfg,
|
||||
mounts: []ReadOnlyMount{{Name: "inputs", HostPath: symlink, Destination: "/workspace/inputs"}},
|
||||
},
|
||||
{
|
||||
name: "missing host",
|
||||
cfg: cfg,
|
||||
mounts: []ReadOnlyMount{{Name: "inputs", HostPath: filepath.Join(host, "missing"), Destination: "/workspace/inputs"}},
|
||||
},
|
||||
{
|
||||
name: "host file",
|
||||
cfg: cfg,
|
||||
mounts: []ReadOnlyMount{{Name: "inputs", HostPath: file, Destination: "/workspace/inputs"}},
|
||||
},
|
||||
{
|
||||
name: "relative destination",
|
||||
cfg: cfg,
|
||||
mounts: []ReadOnlyMount{{Name: "inputs", HostPath: host, Destination: "inputs"}},
|
||||
},
|
||||
{
|
||||
name: "unclean destination",
|
||||
cfg: cfg,
|
||||
mounts: []ReadOnlyMount{{Name: "inputs", HostPath: host, Destination: "/workspace/a/../inputs"}},
|
||||
},
|
||||
{
|
||||
name: "destination equals workdir",
|
||||
cfg: cfg,
|
||||
mounts: []ReadOnlyMount{{Name: "inputs", HostPath: host, Destination: "/workspace"}},
|
||||
},
|
||||
{
|
||||
name: "destination sibling prefix",
|
||||
cfg: cfg,
|
||||
mounts: []ReadOnlyMount{{Name: "inputs", HostPath: host, Destination: "/workspace-other/inputs"}},
|
||||
},
|
||||
{
|
||||
name: "duplicate destination",
|
||||
cfg: cfg,
|
||||
mounts: []ReadOnlyMount{{Name: "a", HostPath: host, Destination: "/workspace/inputs"}, {Name: "b", HostPath: otherHost, Destination: "/workspace/inputs"}},
|
||||
},
|
||||
{
|
||||
name: "invalid workdir",
|
||||
cfg: func() Config {
|
||||
invalid := cfg
|
||||
invalid.Workdir = "workspace"
|
||||
return invalid
|
||||
}(),
|
||||
mounts: []ReadOnlyMount{{Name: "inputs", HostPath: host, Destination: "/workspace/inputs"}},
|
||||
},
|
||||
{
|
||||
name: "workspace access none",
|
||||
cfg: func() Config {
|
||||
none := cfg
|
||||
none.WorkspaceAccess = AccessNone
|
||||
return none
|
||||
}(),
|
||||
mounts: []ReadOnlyMount{{Name: "inputs", HostPath: host, Destination: "/workspace/inputs"}},
|
||||
},
|
||||
{
|
||||
name: "destination contains null",
|
||||
cfg: cfg,
|
||||
mounts: []ReadOnlyMount{{Name: "inputs", HostPath: host, Destination: "/workspace/\x00inputs"}},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, err := validateReadOnlyMounts(tt.cfg, tt.mounts)
|
||||
if err == nil {
|
||||
t.Fatal("validateReadOnlyMounts() error = nil, want rejection")
|
||||
}
|
||||
for _, sensitive := range []string{host, otherHost, file, symlink} {
|
||||
if strings.Contains(err.Error(), sensitive) {
|
||||
t.Fatalf("validation error leaked mount root %q: %v", sensitive, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppendDockerMountArgsAddsSortedReadOnlyMounts(t *testing.T) {
|
||||
workspace := canonicalTempDir(t)
|
||||
alpha := canonicalTempDir(t)
|
||||
zeta := canonicalTempDir(t)
|
||||
cfg := DefaultConfig()
|
||||
|
||||
got, roots, err := appendDockerMountArgs(
|
||||
context.Background(),
|
||||
nil,
|
||||
cfg,
|
||||
workspace,
|
||||
[]ReadOnlyMount{
|
||||
{Name: "zeta", HostPath: zeta, Destination: "/workspace/zeta"},
|
||||
{Name: "alpha", HostPath: alpha, Destination: "/workspace/alpha"},
|
||||
},
|
||||
nil,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("appendDockerMountArgs() error = %v", err)
|
||||
}
|
||||
|
||||
want := []string{
|
||||
"-v", workspace + ":/workspace:rw",
|
||||
"-v", alpha + ":/workspace/alpha:ro",
|
||||
"-v", zeta + ":/workspace/zeta:ro",
|
||||
}
|
||||
if !slices.Equal(got, want) {
|
||||
t.Fatalf("appendDockerMountArgs() = %#v, want %#v", got, want)
|
||||
}
|
||||
for _, root := range []string{workspace, alpha, zeta} {
|
||||
if !slices.Contains(roots, root) {
|
||||
t.Fatalf("sensitive roots %#v do not contain %q", roots, root)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDockerCacheKeyIncludesDeterministicReadOnlyMountSet(t *testing.T) {
|
||||
cfg := DefaultConfig()
|
||||
key := "delegation:9db9"
|
||||
firstHost := canonicalTempDir(t)
|
||||
secondHost := canonicalTempDir(t)
|
||||
first := ReadOnlyMount{Name: "alpha", HostPath: firstHost, Destination: "/workspace/alpha"}
|
||||
second := ReadOnlyMount{Name: "zeta", HostPath: secondHost, Destination: "/workspace/zeta"}
|
||||
|
||||
forward := dockerCacheKey(key, "/workspace/output", cfg, first, second)
|
||||
reverse := dockerCacheKey(key, "/workspace/output", cfg, second, first)
|
||||
if forward != reverse {
|
||||
t.Fatalf("mount ordering changed cache identity: %q != %q", forward, reverse)
|
||||
}
|
||||
|
||||
changedDestination := first
|
||||
changedDestination.Destination = "/workspace/other"
|
||||
if got := dockerCacheKey(key, "/workspace/output", cfg, changedDestination, second); got == forward {
|
||||
t.Fatalf("destination change did not change cache identity: %q", got)
|
||||
}
|
||||
|
||||
changedName := first
|
||||
changedName.Name = "other"
|
||||
if got := dockerCacheKey(key, "/workspace/output", cfg, changedName, second); got == forward {
|
||||
t.Fatalf("name change did not change cache identity: %q", got)
|
||||
}
|
||||
|
||||
changedHost := first
|
||||
changedHost.HostPath = secondHost
|
||||
if got := dockerCacheKey(key, "/workspace/output", cfg, changedHost, second); got == forward {
|
||||
t.Fatalf("host change did not change cache identity: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkspaceAccessOverrideIsPerAcquisition(t *testing.T) {
|
||||
for _, original := range []Access{AccessNone, AccessRO} {
|
||||
cfg := DefaultConfig()
|
||||
cfg.WorkspaceAccess = original
|
||||
opts := ApplyGetOpts([]GetOption{WithWorkspaceAccessOverride(AccessRW)})
|
||||
|
||||
got := applyGetOptsToConfig(cfg, opts)
|
||||
if got.WorkspaceAccess != AccessRW {
|
||||
t.Fatalf("override from %q = %q, want rw", original, got.WorkspaceAccess)
|
||||
}
|
||||
if cfg.WorkspaceAccess != original {
|
||||
t.Fatalf("override mutated shared config: got %q, want %q", cfg.WorkspaceAccess, original)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyGetOptsCopiesMounts(t *testing.T) {
|
||||
mounts := []ReadOnlyMount{{Name: "inputs", HostPath: "/canonical/host", Destination: "/workspace/inputs"}}
|
||||
opts := ApplyGetOpts([]GetOption{WithReadOnlyMounts(mounts...)})
|
||||
mounts[0].Name = "mutated"
|
||||
|
||||
if got := opts.ReadOnlyMounts[0].Name; got != "inputs" {
|
||||
t.Fatalf("ApplyGetOpts retained caller slice: name = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDockerManagerReleaseRetainsSandboxWhenDestroyFails(t *testing.T) {
|
||||
destroyErr := errors.New("destroy failed")
|
||||
sb := &DockerSandbox{
|
||||
containerID: "container-1",
|
||||
destroyFunc: func(context.Context) error {
|
||||
return destroyErr
|
||||
},
|
||||
}
|
||||
manager := &DockerManager{sandboxes: map[string]*DockerSandbox{"w123:delegation": sb}}
|
||||
|
||||
err := manager.Release(context.Background(), "delegation")
|
||||
if !errors.Is(err, destroyErr) {
|
||||
t.Fatalf("Release() error = %v, want %v", err, destroyErr)
|
||||
}
|
||||
if got := manager.Stats()["active"]; got != 1 {
|
||||
t.Fatalf("active sandboxes after failed release = %v, want 1", got)
|
||||
}
|
||||
|
||||
sb.destroyFunc = func(context.Context) error { return nil }
|
||||
if err := manager.Release(context.Background(), "delegation"); err != nil {
|
||||
t.Fatalf("Release() retry error = %v", err)
|
||||
}
|
||||
if got := manager.Stats()["active"]; got != 0 {
|
||||
t.Fatalf("active sandboxes after successful release = %v, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDockerManagerReleaseFailureRemainsVisibleConcurrently(t *testing.T) {
|
||||
destroyStarted := make(chan struct{})
|
||||
allowDestroy := make(chan struct{})
|
||||
destroyErr := errors.New("destroy failed")
|
||||
sb := &DockerSandbox{
|
||||
containerID: "container-1",
|
||||
destroyFunc: func(context.Context) error {
|
||||
close(destroyStarted)
|
||||
<-allowDestroy
|
||||
return destroyErr
|
||||
},
|
||||
}
|
||||
manager := &DockerManager{sandboxes: map[string]*DockerSandbox{"delegation": sb}}
|
||||
|
||||
releaseDone := make(chan error, 1)
|
||||
go func() {
|
||||
releaseDone <- manager.Release(context.Background(), "delegation")
|
||||
}()
|
||||
<-destroyStarted
|
||||
|
||||
statsStarted := make(chan struct{})
|
||||
statsDone := make(chan map[string]any, 1)
|
||||
go func() {
|
||||
close(statsStarted)
|
||||
statsDone <- manager.Stats()
|
||||
}()
|
||||
<-statsStarted
|
||||
|
||||
select {
|
||||
case <-statsDone:
|
||||
t.Fatal("Stats returned while Release was still deciding container lifecycle")
|
||||
case <-time.After(25 * time.Millisecond):
|
||||
}
|
||||
|
||||
close(allowDestroy)
|
||||
if err := <-releaseDone; !errors.Is(err, destroyErr) {
|
||||
t.Fatalf("Release() error = %v, want %v", err, destroyErr)
|
||||
}
|
||||
select {
|
||||
case stats := <-statsDone:
|
||||
if got := stats["active"]; got != 1 {
|
||||
t.Fatalf("concurrent active sandboxes = %v, want 1", got)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Stats did not return after Release completed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedactMountRootsRemovesSensitivePaths(t *testing.T) {
|
||||
roots := []string{"/private/runtime/delegation", "/private/runtime/delegation/inputs"}
|
||||
message := "invalid mount /private/runtime/delegation/inputs at /private/runtime/delegation"
|
||||
got := redactMountRoots(message, roots)
|
||||
|
||||
for _, root := range roots {
|
||||
if strings.Contains(got, root) {
|
||||
t.Fatalf("redactMountRoots() leaked %q in %q", root, got)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(got, redactedMountRoot) {
|
||||
t.Fatalf("redactMountRoots() = %q, want redaction marker", got)
|
||||
}
|
||||
}
|
||||
|
||||
func canonicalTempDir(t *testing.T) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
canonical, err := filepath.EvalSymlinks(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("canonicalize temp dir: %v", err)
|
||||
}
|
||||
return canonical
|
||||
}
|
||||
@@ -27,7 +27,7 @@ func resolveHostWorkspacePath(ctx context.Context, localPath string) string {
|
||||
|
||||
containerID := detectContainerID()
|
||||
if containerID == "" {
|
||||
slog.Error("sandbox.resolve: cannot determine container ID — DooD volume mounts will fail", "path", localPath)
|
||||
slog.Error("sandbox.resolve: cannot determine container ID — DooD volume mounts will fail")
|
||||
return localPath
|
||||
}
|
||||
|
||||
@@ -60,10 +60,7 @@ func resolveHostWorkspacePath(ctx context.Context, localPath string) string {
|
||||
if resolved, err := filepath.EvalSymlinks(localPath); err == nil {
|
||||
resolvedClean := filepath.Clean(resolved)
|
||||
if resolvedClean != targetDir {
|
||||
slog.Info("sandbox.resolve: symlink resolved",
|
||||
"path", localPath,
|
||||
"resolved", resolvedClean,
|
||||
)
|
||||
slog.Debug("sandbox.resolve: canonicalized local mount path")
|
||||
}
|
||||
targetDir = resolvedClean
|
||||
}
|
||||
@@ -88,7 +85,7 @@ func resolveHostWorkspacePath(ctx context.Context, localPath string) string {
|
||||
}
|
||||
|
||||
if bestDest == "" {
|
||||
slog.Warn("sandbox.resolve: no matching mount found", "path", localPath, "container", containerID)
|
||||
slog.Warn("sandbox.resolve: no matching mount found", "container", containerID)
|
||||
return localPath
|
||||
}
|
||||
|
||||
@@ -96,7 +93,7 @@ func resolveHostWorkspacePath(ctx context.Context, localPath string) string {
|
||||
// use the host source path (assumes local volume driver).
|
||||
if bestType == "volume" && bestName != "" {
|
||||
if bestRel == "." {
|
||||
slog.Debug("sandbox.resolve: resolved to named volume", "path", localPath, "volume", bestName)
|
||||
slog.Debug("sandbox.resolve: resolved mount to named volume")
|
||||
return bestName
|
||||
}
|
||||
if bestSource != "" {
|
||||
@@ -107,11 +104,11 @@ func resolveHostWorkspacePath(ctx context.Context, localPath string) string {
|
||||
// Bind mount: join source with relative path.
|
||||
if bestSource != "" {
|
||||
resolved := filepath.Join(bestSource, bestRel)
|
||||
slog.Debug("sandbox.resolve: resolved to host path", "path", localPath, "host", resolved)
|
||||
slog.Debug("sandbox.resolve: resolved mount to host path")
|
||||
return resolved
|
||||
}
|
||||
|
||||
slog.Warn("sandbox.resolve: mount found but no source path", "path", localPath, "mount", bestDest)
|
||||
slog.Warn("sandbox.resolve: mount found but no source path")
|
||||
return localPath
|
||||
}
|
||||
|
||||
|
||||
@@ -181,6 +181,61 @@ func ApplyExecOpts(opts []ExecOption) ExecOpts {
|
||||
return o
|
||||
}
|
||||
|
||||
// ReadOnlyMount describes an additional runtime-only bind mount.
|
||||
// HostPath must be an absolute, canonical host path. Destination must be an
|
||||
// absolute canonical container path strictly beneath the configured workdir.
|
||||
type ReadOnlyMount struct {
|
||||
Name string
|
||||
HostPath string
|
||||
Destination string
|
||||
}
|
||||
|
||||
// GetOption configures optional behavior for Manager.Get calls.
|
||||
type GetOption func(*GetOpts)
|
||||
|
||||
// GetOpts holds optional settings applied via GetOption.
|
||||
type GetOpts struct {
|
||||
ReadOnlyMounts []ReadOnlyMount
|
||||
WorkspaceAccessOverride *Access
|
||||
}
|
||||
|
||||
// WithReadOnlyMounts adds runtime-only read-only mounts to a sandbox.
|
||||
func WithReadOnlyMounts(mounts ...ReadOnlyMount) GetOption {
|
||||
copied := append([]ReadOnlyMount(nil), mounts...)
|
||||
return func(o *GetOpts) {
|
||||
o.ReadOnlyMounts = append(o.ReadOnlyMounts, copied...)
|
||||
}
|
||||
}
|
||||
|
||||
// WithWorkspaceAccessOverride changes access for this isolated sandbox
|
||||
// instance only. Callers must validate the workspace authority before using it.
|
||||
func WithWorkspaceAccessOverride(access Access) GetOption {
|
||||
return func(o *GetOpts) {
|
||||
override := access
|
||||
o.WorkspaceAccessOverride = &override
|
||||
}
|
||||
}
|
||||
|
||||
// ApplyGetOpts resolves variadic GetOption values without retaining caller-owned
|
||||
// slices.
|
||||
func ApplyGetOpts(opts []GetOption) GetOpts {
|
||||
var o GetOpts
|
||||
for _, opt := range opts {
|
||||
if opt != nil {
|
||||
opt(&o)
|
||||
}
|
||||
}
|
||||
o.ReadOnlyMounts = append([]ReadOnlyMount(nil), o.ReadOnlyMounts...)
|
||||
return o
|
||||
}
|
||||
|
||||
func applyGetOptsToConfig(cfg Config, opts GetOpts) Config {
|
||||
if opts.WorkspaceAccessOverride != nil {
|
||||
cfg.WorkspaceAccess = *opts.WorkspaceAccessOverride
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// Sandbox is the interface for sandboxed code execution.
|
||||
type Sandbox interface {
|
||||
// Exec runs a command inside the sandbox and returns the result.
|
||||
@@ -203,7 +258,7 @@ type Manager interface {
|
||||
// For agent scope: key = agentID
|
||||
// For shared scope: key = "shared"
|
||||
// If cfgOverride is non-nil, it is used instead of the global config for new containers.
|
||||
Get(ctx context.Context, key string, workspace string, cfgOverride *Config) (Sandbox, error)
|
||||
Get(ctx context.Context, key string, workspace string, cfgOverride *Config, opts ...GetOption) (Sandbox, error)
|
||||
|
||||
// Release destroys a sandbox by key.
|
||||
Release(ctx context.Context, key string) error
|
||||
|
||||
@@ -25,7 +25,7 @@ type AgentLinkData struct {
|
||||
BaseModel
|
||||
SourceAgentID uuid.UUID `json:"source_agent_id" db:"source_agent_id"`
|
||||
TargetAgentID uuid.UUID `json:"target_agent_id" db:"target_agent_id"`
|
||||
Direction string `json:"direction" db:"direction"` // "outbound", "inbound", "bidirectional"
|
||||
Direction string `json:"direction" db:"direction"` // "outbound", "inbound", "bidirectional"
|
||||
TeamID *uuid.UUID `json:"team_id,omitempty" db:"team_id"` // non-nil = auto-created by team
|
||||
Description string `json:"description,omitempty" db:"description"`
|
||||
MaxConcurrent int `json:"max_concurrent" db:"max_concurrent"`
|
||||
@@ -34,13 +34,13 @@ type AgentLinkData struct {
|
||||
CreatedBy string `json:"created_by" db:"created_by"`
|
||||
|
||||
// Joined fields (populated by queries that JOIN agents table)
|
||||
SourceAgentKey string `json:"source_agent_key,omitempty" db:"source_agent_key"`
|
||||
SourceDisplayName string `json:"source_display_name,omitempty" db:"source_display_name"`
|
||||
SourceEmoji string `json:"source_emoji,omitempty" db:"source_emoji"`
|
||||
TargetAgentKey string `json:"target_agent_key,omitempty" db:"target_agent_key"`
|
||||
TargetDisplayName string `json:"target_display_name,omitempty" db:"target_display_name"`
|
||||
TargetEmoji string `json:"target_emoji,omitempty" db:"target_emoji"`
|
||||
TargetDescription string `json:"target_description,omitempty" db:"target_description"`
|
||||
SourceAgentKey string `json:"source_agent_key,omitempty" db:"source_agent_key"`
|
||||
SourceDisplayName string `json:"source_display_name,omitempty" db:"source_display_name"`
|
||||
SourceEmoji string `json:"source_emoji,omitempty" db:"source_emoji"`
|
||||
TargetAgentKey string `json:"target_agent_key,omitempty" db:"target_agent_key"`
|
||||
TargetDisplayName string `json:"target_display_name,omitempty" db:"target_display_name"`
|
||||
TargetEmoji string `json:"target_emoji,omitempty" db:"target_emoji"`
|
||||
TargetDescription string `json:"target_description,omitempty" db:"target_description"`
|
||||
TeamName string `json:"team_name,omitempty" db:"team_name"` // from LEFT JOIN agent_teams (link's own team)
|
||||
TargetIsTeamLead bool `json:"target_is_team_lead,omitempty" db:"target_is_team_lead"` // true if target is lead of any active team
|
||||
TargetTeamName string `json:"target_team_name,omitempty" db:"target_team_name"` // name of team the target leads
|
||||
@@ -62,7 +62,9 @@ type AgentLinkStore interface {
|
||||
// CanDelegate checks if fromAgent can delegate to toAgent considering direction.
|
||||
CanDelegate(ctx context.Context, fromAgentID, toAgentID uuid.UUID) (bool, error)
|
||||
|
||||
// GetLinkBetween returns the active link allowing fromAgent to delegate to toAgent.
|
||||
// GetLinkBetween returns the active directional link allowing fromAgent to
|
||||
// delegate to toAgent. MaxConcurrent is retained for API compatibility but
|
||||
// is reserved and has no runtime admission effect.
|
||||
// Returns full link data including Settings for per-user permission checks.
|
||||
// Returns nil, nil if no matching link exists.
|
||||
GetLinkBetween(ctx context.Context, fromAgentID, toAgentID uuid.UUID) (*AgentLinkData, error)
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
@@ -257,10 +258,15 @@ func (s *PGAgentLinkStore) GetLinkBetween(ctx context.Context, fromAgentID, toAg
|
||||
(source_agent_id = $1 AND target_agent_id = $2 AND direction IN ('outbound', 'bidirectional'))
|
||||
OR
|
||||
(source_agent_id = $2 AND target_agent_id = $1 AND direction IN ('inbound', 'bidirectional'))
|
||||
) LIMIT 1`, args...)
|
||||
)
|
||||
ORDER BY CASE WHEN source_agent_id = $1 THEN 0 ELSE 1 END, created_at, id
|
||||
LIMIT 1`, args...)
|
||||
d, err := scanLinkRow(row)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil // no link found
|
||||
return nil, err
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ func NewPGStores(cfg store.StoreConfig) (*store.Stores, error) {
|
||||
if skillsDir == "" {
|
||||
skillsDir = config.ResolvedDataDirFromEnv() + "/skills-store"
|
||||
}
|
||||
subagentTasks := NewPGSubagentTaskStore(db)
|
||||
|
||||
pgStores := &store.Stores{
|
||||
DB: db,
|
||||
@@ -59,7 +60,8 @@ func NewPGStores(cfg store.StoreConfig) (*store.Stores, error) {
|
||||
SkillTenantCfgs: NewPGSkillTenantConfigStore(db),
|
||||
SkillEvolution: NewPGSkillEvolutionStore(db),
|
||||
SystemConfigs: NewPGSystemConfigStore(db),
|
||||
SubagentTasks: NewPGSubagentTaskStore(db),
|
||||
SubagentTasks: subagentTasks,
|
||||
SubagentTaskRecovery: subagentTasks,
|
||||
Vault: NewPGVaultStore(db),
|
||||
Episodic: NewPGEpisodicStore(db),
|
||||
EvolutionMetrics: NewPGEvolutionMetricsStore(db),
|
||||
|
||||
@@ -18,18 +18,23 @@ type PGSubagentTaskStore struct {
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
var _ store.SubagentTaskRecoveryStore = (*PGSubagentTaskStore)(nil)
|
||||
|
||||
// NewPGSubagentTaskStore creates a new PostgreSQL-backed subagent task store.
|
||||
func NewPGSubagentTaskStore(db *sql.DB) *PGSubagentTaskStore {
|
||||
return &PGSubagentTaskStore{db: db}
|
||||
}
|
||||
|
||||
const subagentTaskInsertCols = `tenant_id, parent_agent_key, session_key, subject, description,
|
||||
const subagentTaskInsertCols = `tenant_id, root_agent_id, parent_agent_key, session_key, subject, description,
|
||||
status, result, depth, model, provider, iterations, input_tokens, output_tokens,
|
||||
origin_channel, origin_chat_id, origin_peer_kind, origin_user_id, spawned_by, metadata`
|
||||
|
||||
// Create persists a new subagent task at spawn time.
|
||||
func (s *PGSubagentTaskStore) Create(ctx context.Context, task *store.SubagentTaskData) error {
|
||||
tid := tenantIDForInsert(ctx)
|
||||
if task.RootAgentID == uuid.Nil {
|
||||
return store.ErrSubagentRootAgentIDRequired
|
||||
}
|
||||
|
||||
metaJSON := []byte("{}")
|
||||
if len(task.Metadata) > 0 {
|
||||
@@ -39,11 +44,11 @@ func (s *PGSubagentTaskStore) Create(ctx context.Context, task *store.SubagentTa
|
||||
}
|
||||
|
||||
q := fmt.Sprintf(`INSERT INTO subagent_tasks (id, %s)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21)
|
||||
ON CONFLICT (id) DO NOTHING`, subagentTaskInsertCols)
|
||||
|
||||
_, err := s.db.ExecContext(ctx, q,
|
||||
task.ID, tid, task.ParentAgentKey, task.SessionKey, task.Subject, task.Description,
|
||||
task.ID, tid, task.RootAgentID, task.ParentAgentKey, task.SessionKey, task.Subject, task.Description,
|
||||
task.Status, task.Result, task.Depth, task.Model, task.Provider,
|
||||
task.Iterations, task.InputTokens, task.OutputTokens,
|
||||
task.OriginChannel, task.OriginChatID, task.OriginPeerKind, task.OriginUserID,
|
||||
@@ -52,7 +57,7 @@ func (s *PGSubagentTaskStore) Create(ctx context.Context, task *store.SubagentTa
|
||||
return err
|
||||
}
|
||||
|
||||
const subagentTaskSelectCols = `id, tenant_id, parent_agent_key, session_key, subject, description,
|
||||
const subagentTaskSelectCols = `id, tenant_id, root_agent_id, parent_agent_key, session_key, subject, description,
|
||||
status, result, depth, model, provider, iterations, input_tokens, output_tokens,
|
||||
origin_channel, origin_chat_id, origin_peer_kind, origin_user_id, spawned_by,
|
||||
completed_at, archived_at, COALESCE(metadata, '{}'), created_at, updated_at`
|
||||
@@ -62,7 +67,7 @@ func scanTask(row interface{ Scan(...any) error }) (*store.SubagentTaskData, err
|
||||
var t store.SubagentTaskData
|
||||
var metaJSON []byte
|
||||
err := row.Scan(
|
||||
&t.ID, &t.TenantID, &t.ParentAgentKey, &t.SessionKey, &t.Subject, &t.Description,
|
||||
&t.ID, &t.TenantID, &t.RootAgentID, &t.ParentAgentKey, &t.SessionKey, &t.Subject, &t.Description,
|
||||
&t.Status, &t.Result, &t.Depth, &t.Model, &t.Provider,
|
||||
&t.Iterations, &t.InputTokens, &t.OutputTokens,
|
||||
&t.OriginChannel, &t.OriginChatID, &t.OriginPeerKind, &t.OriginUserID, &t.SpawnedBy,
|
||||
@@ -77,14 +82,20 @@ func scanTask(row interface{ Scan(...any) error }) (*store.SubagentTaskData, err
|
||||
return &t, nil
|
||||
}
|
||||
|
||||
// Get retrieves a single task by ID (tenant-scoped).
|
||||
func (s *PGSubagentTaskStore) Get(ctx context.Context, id uuid.UUID) (*store.SubagentTaskData, error) {
|
||||
// Get retrieves a task owned by the tenant and immutable root-agent UUID.
|
||||
func (s *PGSubagentTaskStore) Get(
|
||||
ctx context.Context, rootAgentID, id uuid.UUID,
|
||||
) (*store.SubagentTaskData, error) {
|
||||
tid, err := requireTenantID(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
q := fmt.Sprintf(`SELECT %s FROM subagent_tasks WHERE id = $1 AND tenant_id = $2`, subagentTaskSelectCols)
|
||||
row := s.db.QueryRowContext(ctx, q, id, tid)
|
||||
if rootAgentID == uuid.Nil {
|
||||
return nil, store.ErrSubagentRootAgentIDRequired
|
||||
}
|
||||
q := fmt.Sprintf(`SELECT %s FROM subagent_tasks
|
||||
WHERE id = $1 AND tenant_id = $2 AND root_agent_id = $3`, subagentTaskSelectCols)
|
||||
row := s.db.QueryRowContext(ctx, q, id, tid, rootAgentID)
|
||||
t, err := scanTask(row)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
@@ -94,7 +105,7 @@ func (s *PGSubagentTaskStore) Get(ctx context.Context, id uuid.UUID) (*store.Sub
|
||||
|
||||
// UpdateStatus updates status, result, iterations, and token counts.
|
||||
func (s *PGSubagentTaskStore) UpdateStatus(
|
||||
ctx context.Context, id uuid.UUID,
|
||||
ctx context.Context, rootAgentID, id uuid.UUID,
|
||||
status string, result *string, iterations int,
|
||||
inputTokens, outputTokens int64,
|
||||
) error {
|
||||
@@ -102,9 +113,12 @@ func (s *PGSubagentTaskStore) UpdateStatus(
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rootAgentID == uuid.Nil {
|
||||
return store.ErrSubagentRootAgentIDRequired
|
||||
}
|
||||
|
||||
var completedAt *time.Time
|
||||
if status != "running" {
|
||||
if store.IsTerminalSubagentTaskStatus(status) {
|
||||
now := time.Now().UTC()
|
||||
completedAt = &now
|
||||
}
|
||||
@@ -113,34 +127,49 @@ func (s *PGSubagentTaskStore) UpdateStatus(
|
||||
status = $1, result = $2, iterations = $3,
|
||||
input_tokens = $4, output_tokens = $5,
|
||||
completed_at = $6, updated_at = NOW()
|
||||
WHERE id = $7 AND tenant_id = $8`
|
||||
_, err = s.db.ExecContext(ctx, q,
|
||||
WHERE id = $7 AND tenant_id = $8 AND root_agent_id = $9`
|
||||
res, err := s.db.ExecContext(ctx, q,
|
||||
status, result, iterations, inputTokens, outputTokens,
|
||||
completedAt, id, tid,
|
||||
completedAt, id, tid, rootAgentID,
|
||||
)
|
||||
return err
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
affected, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected != 1 {
|
||||
return store.ErrSubagentTaskNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListByParent returns tasks for a parent agent key, optionally filtered by status.
|
||||
// ListByParent returns tasks for a root-agent UUID, optionally filtered by status.
|
||||
func (s *PGSubagentTaskStore) ListByParent(
|
||||
ctx context.Context, parentAgentKey string, statusFilter string,
|
||||
ctx context.Context, rootAgentID uuid.UUID, statusFilter string,
|
||||
) ([]store.SubagentTaskData, error) {
|
||||
tid, err := requireTenantID(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if rootAgentID == uuid.Nil {
|
||||
return nil, store.ErrSubagentRootAgentIDRequired
|
||||
}
|
||||
|
||||
var rows *sql.Rows
|
||||
if statusFilter != "" {
|
||||
q := fmt.Sprintf(`SELECT %s FROM subagent_tasks
|
||||
WHERE tenant_id = $1 AND parent_agent_key = $2 AND status = $3
|
||||
ORDER BY created_at DESC LIMIT 50`, subagentTaskSelectCols)
|
||||
rows, err = s.db.QueryContext(ctx, q, tid, parentAgentKey, statusFilter)
|
||||
WHERE tenant_id = $1 AND root_agent_id = $2 AND status = $3
|
||||
AND COALESCE(metadata->>'completion_kind', 'subagent') <> 'delegate'
|
||||
ORDER BY created_at DESC LIMIT 50`, subagentTaskSelectCols)
|
||||
rows, err = s.db.QueryContext(ctx, q, tid, rootAgentID, statusFilter)
|
||||
} else {
|
||||
q := fmt.Sprintf(`SELECT %s FROM subagent_tasks
|
||||
WHERE tenant_id = $1 AND parent_agent_key = $2
|
||||
ORDER BY created_at DESC LIMIT 50`, subagentTaskSelectCols)
|
||||
rows, err = s.db.QueryContext(ctx, q, tid, parentAgentKey)
|
||||
WHERE tenant_id = $1 AND root_agent_id = $2
|
||||
AND COALESCE(metadata->>'completion_kind', 'subagent') <> 'delegate'
|
||||
ORDER BY created_at DESC LIMIT 50`, subagentTaskSelectCols)
|
||||
rows, err = s.db.QueryContext(ctx, q, tid, rootAgentID)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -152,17 +181,21 @@ func (s *PGSubagentTaskStore) ListByParent(
|
||||
|
||||
// ListBySession returns tasks for a specific session key (tenant-scoped).
|
||||
func (s *PGSubagentTaskStore) ListBySession(
|
||||
ctx context.Context, sessionKey string,
|
||||
ctx context.Context, rootAgentID uuid.UUID, sessionKey string,
|
||||
) ([]store.SubagentTaskData, error) {
|
||||
tid, err := requireTenantID(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if rootAgentID == uuid.Nil {
|
||||
return nil, store.ErrSubagentRootAgentIDRequired
|
||||
}
|
||||
|
||||
q := fmt.Sprintf(`SELECT %s FROM subagent_tasks
|
||||
WHERE tenant_id = $1 AND session_key = $2
|
||||
WHERE tenant_id = $1 AND root_agent_id = $2 AND session_key = $3
|
||||
AND COALESCE(metadata->>'completion_kind', 'subagent') <> 'delegate'
|
||||
ORDER BY created_at DESC LIMIT 50`, subagentTaskSelectCols)
|
||||
rows, err := s.db.QueryContext(ctx, q, tid, sessionKey)
|
||||
rows, err := s.db.QueryContext(ctx, q, tid, rootAgentID, sessionKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -171,25 +204,54 @@ func (s *PGSubagentTaskStore) ListBySession(
|
||||
return collectTasks(rows)
|
||||
}
|
||||
|
||||
// Archive marks old completed/failed/cancelled tasks as archived.
|
||||
func (s *PGSubagentTaskStore) Archive(ctx context.Context, olderThan time.Duration) (int64, error) {
|
||||
// Archive marks a bounded batch of old terminal tasks as archived.
|
||||
func (s *PGSubagentTaskStore) Archive(
|
||||
ctx context.Context, rootAgentID uuid.UUID, olderThan time.Duration, limit int,
|
||||
) (int64, error) {
|
||||
tid, err := requireTenantID(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if rootAgentID == uuid.Nil {
|
||||
return 0, store.ErrSubagentRootAgentIDRequired
|
||||
}
|
||||
if limit <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
cutoff := time.Now().UTC().Add(-olderThan)
|
||||
q := `UPDATE subagent_tasks SET archived_at = NOW(), updated_at = NOW()
|
||||
WHERE status IN ('completed', 'failed', 'cancelled')
|
||||
AND archived_at IS NULL AND completed_at < $1`
|
||||
res, err := s.db.ExecContext(ctx, q, cutoff)
|
||||
q := `WITH candidates AS (
|
||||
SELECT id
|
||||
FROM subagent_tasks
|
||||
WHERE tenant_id = $1 AND root_agent_id = $2
|
||||
AND status IN ('completed', 'failed', 'cancelled')
|
||||
AND archived_at IS NULL AND completed_at < $3
|
||||
ORDER BY completed_at, id
|
||||
LIMIT $4
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
UPDATE subagent_tasks AS task
|
||||
SET archived_at = NOW(), updated_at = NOW()
|
||||
FROM candidates
|
||||
WHERE task.id = candidates.id`
|
||||
res, err := s.db.ExecContext(ctx, q, tid, rootAgentID, cutoff, limit)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return res.RowsAffected()
|
||||
}
|
||||
|
||||
// UpdateMetadata merges metadata on an existing task.
|
||||
func (s *PGSubagentTaskStore) UpdateMetadata(ctx context.Context, id uuid.UUID, metadata map[string]any) error {
|
||||
// UpdateMetadata merges metadata on a task owned by the tenant and root agent.
|
||||
func (s *PGSubagentTaskStore) UpdateMetadata(
|
||||
ctx context.Context, rootAgentID, id uuid.UUID, metadata map[string]any,
|
||||
) error {
|
||||
tid, err := requireTenantID(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rootAgentID == uuid.Nil {
|
||||
return store.ErrSubagentRootAgentIDRequired
|
||||
}
|
||||
|
||||
metaJSON, err := json.Marshal(metadata)
|
||||
if err != nil {
|
||||
@@ -197,9 +259,35 @@ func (s *PGSubagentTaskStore) UpdateMetadata(ctx context.Context, id uuid.UUID,
|
||||
}
|
||||
|
||||
q := `UPDATE subagent_tasks SET metadata = metadata || $1, updated_at = NOW()
|
||||
WHERE id = $2 AND tenant_id = $3`
|
||||
_, err = s.db.ExecContext(ctx, q, metaJSON, id, tid)
|
||||
return err
|
||||
WHERE id = $2 AND tenant_id = $3 AND root_agent_id = $4`
|
||||
res, err := s.db.ExecContext(ctx, q, metaJSON, id, tid, rootAgentID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
affected, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected != 1 {
|
||||
return store.ErrSubagentTaskNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RecoverInterrupted marks child runs that cannot survive a process restart as
|
||||
// failed. This cross-tenant update runs before the gateway accepts new work.
|
||||
func (s *PGSubagentTaskStore) RecoverInterrupted(ctx context.Context) (int64, error) {
|
||||
if !store.IsMasterScope(ctx) {
|
||||
return 0, fmt.Errorf("recover interrupted subagent tasks requires master scope")
|
||||
}
|
||||
q := `UPDATE subagent_tasks SET
|
||||
status = 'failed', result = $1, completed_at = NOW(), updated_at = NOW()
|
||||
WHERE status NOT IN ('completed', 'failed', 'cancelled')`
|
||||
res, err := s.db.ExecContext(ctx, q, store.InterruptedSubagentTaskResult)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return res.RowsAffected()
|
||||
}
|
||||
|
||||
// collectTasks scans rows into a slice.
|
||||
|
||||
@@ -0,0 +1,472 @@
|
||||
package pg
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
)
|
||||
|
||||
func createPGSubagentTask(
|
||||
t *testing.T,
|
||||
taskStore *PGSubagentTaskStore,
|
||||
ctx context.Context,
|
||||
rootAgentID uuid.UUID,
|
||||
rootAgentKey, sessionKey, status string,
|
||||
) uuid.UUID {
|
||||
t.Helper()
|
||||
|
||||
id := uuid.Must(uuid.NewV7())
|
||||
task := &store.SubagentTaskData{
|
||||
RootAgentID: rootAgentID,
|
||||
ParentAgentKey: rootAgentKey,
|
||||
SessionKey: &sessionKey,
|
||||
Subject: "store test",
|
||||
Description: "verify scoped persistence",
|
||||
Status: status,
|
||||
Depth: 1,
|
||||
Metadata: map[string]any{},
|
||||
}
|
||||
task.ID = id
|
||||
if err := taskStore.Create(ctx, task); err != nil {
|
||||
t.Fatalf("Create(%s): %v", id, err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func TestPGSubagentTaskStoreRequiresTenantAndRootScope(t *testing.T) {
|
||||
db := hooksTestDB(t)
|
||||
tenantA, rootAID := seedTenantAndAgent(t, db)
|
||||
tenantB, tenantBRootID := seedTenantAndAgent(t, db)
|
||||
ctxA := tenantScopedCtx(tenantA)
|
||||
ctxB := tenantScopedCtx(tenantB)
|
||||
taskStore := NewPGSubagentTaskStore(db)
|
||||
|
||||
const (
|
||||
rootA = "root-a"
|
||||
rootB = "root-b"
|
||||
sessionID = "shared-session"
|
||||
)
|
||||
rootBID := uuid.Must(uuid.NewV7())
|
||||
if _, err := db.Exec(
|
||||
`INSERT INTO agents (id, tenant_id, agent_key, agent_type, status, provider, model, owner_id)
|
||||
VALUES ($1,$2,$3,'predefined','active','test','test-model','owner')`,
|
||||
rootBID, tenantA, rootB,
|
||||
); err != nil {
|
||||
t.Fatalf("seed second root agent: %v", err)
|
||||
}
|
||||
taskA := createPGSubagentTask(t, taskStore, ctxA, rootAID, rootA, sessionID, "queued")
|
||||
taskB := createPGSubagentTask(t, taskStore, ctxA, rootBID, rootB, sessionID, "queued")
|
||||
delegationTask := createPGSubagentTask(t, taskStore, ctxA, rootAID, rootA, sessionID, "queued")
|
||||
if err := taskStore.UpdateMetadata(ctxA, rootAID, delegationTask, map[string]any{
|
||||
"completion_kind": "delegate",
|
||||
}); err != nil {
|
||||
t.Fatalf("mark delegation completion: %v", err)
|
||||
}
|
||||
_ = createPGSubagentTask(t, taskStore, ctxB, tenantBRootID, rootA, sessionID, "queued")
|
||||
crossTenantTask := &store.SubagentTaskData{
|
||||
BaseModel: store.BaseModel{ID: uuid.Must(uuid.NewV7())},
|
||||
RootAgentID: tenantBRootID,
|
||||
ParentAgentKey: rootA,
|
||||
Subject: "cross tenant",
|
||||
Description: "must be rejected",
|
||||
Status: "queued",
|
||||
Depth: 1,
|
||||
}
|
||||
if err := taskStore.Create(ctxA, crossTenantTask); err == nil {
|
||||
t.Fatal("Create accepted root agent from another tenant")
|
||||
}
|
||||
|
||||
got, err := taskStore.Get(ctxA, rootAID, taskA)
|
||||
if err != nil {
|
||||
t.Fatalf("Get owning scope: %v", err)
|
||||
}
|
||||
if got == nil || got.ID != taskA {
|
||||
t.Fatalf("Get owning scope = %#v, want task %s", got, taskA)
|
||||
}
|
||||
|
||||
got, err = taskStore.Get(ctxA, rootBID, taskA)
|
||||
if err != nil {
|
||||
t.Fatalf("Get cross-root: %v", err)
|
||||
}
|
||||
if got != nil {
|
||||
t.Fatalf("Get cross-root = %#v, want nil", got)
|
||||
}
|
||||
|
||||
got, err = taskStore.Get(ctxB, rootAID, taskA)
|
||||
if err != nil {
|
||||
t.Fatalf("Get cross-tenant: %v", err)
|
||||
}
|
||||
if got != nil {
|
||||
t.Fatalf("Get cross-tenant = %#v, want nil", got)
|
||||
}
|
||||
|
||||
if err := taskStore.UpdateStatus(ctxA, rootBID, taskA, "completed", nil, 3, 10, 20); !errors.Is(err, store.ErrSubagentTaskNotFound) {
|
||||
t.Fatalf("UpdateStatus cross-root error = %v, want scoped not found", err)
|
||||
}
|
||||
got, err = taskStore.Get(ctxA, rootAID, taskA)
|
||||
if err != nil {
|
||||
t.Fatalf("Get after cross-root status update: %v", err)
|
||||
}
|
||||
if got.Status != "queued" || got.CompletedAt != nil {
|
||||
t.Fatalf("cross-root status update changed task: status=%q completed_at=%v", got.Status, got.CompletedAt)
|
||||
}
|
||||
|
||||
if err := taskStore.UpdateMetadata(ctxA, rootBID, taskA, map[string]any{"denied": true}); !errors.Is(err, store.ErrSubagentTaskNotFound) {
|
||||
t.Fatalf("UpdateMetadata cross-root error = %v, want scoped not found", err)
|
||||
}
|
||||
got, err = taskStore.Get(ctxA, rootAID, taskA)
|
||||
if err != nil {
|
||||
t.Fatalf("Get after cross-root metadata update: %v", err)
|
||||
}
|
||||
if _, exists := got.Metadata["denied"]; exists {
|
||||
t.Fatalf("cross-root metadata update changed task: %#v", got.Metadata)
|
||||
}
|
||||
if err := taskStore.UpdateMetadata(ctxA, rootAID, taskA, map[string]any{
|
||||
"announcement_status": "undelivered",
|
||||
"delivered": false,
|
||||
}); err != nil {
|
||||
t.Fatalf("UpdateMetadata owning scope: %v", err)
|
||||
}
|
||||
got, err = taskStore.Get(ctxA, rootAID, taskA)
|
||||
if err != nil {
|
||||
t.Fatalf("Get after owning metadata update: %v", err)
|
||||
}
|
||||
if got.Metadata["announcement_status"] != "undelivered" || got.Metadata["delivered"] != false {
|
||||
t.Fatalf("metadata JSON types were not preserved: %#v", got.Metadata)
|
||||
}
|
||||
|
||||
parentTasksA, err := taskStore.ListByParent(ctxA, rootAID, "")
|
||||
if err != nil {
|
||||
t.Fatalf("ListByParent root A: %v", err)
|
||||
}
|
||||
if len(parentTasksA) != 1 || parentTasksA[0].ID != taskA {
|
||||
t.Fatalf("ListByParent root A = %#v, want only self-clone %s", parentTasksA, taskA)
|
||||
}
|
||||
parentTasksB, err := taskStore.ListByParent(ctxA, rootBID, "queued")
|
||||
if err != nil {
|
||||
t.Fatalf("ListByParent root B: %v", err)
|
||||
}
|
||||
if len(parentTasksB) != 1 || parentTasksB[0].ID != taskB {
|
||||
t.Fatalf("ListByParent root B = %#v, want only %s", parentTasksB, taskB)
|
||||
}
|
||||
|
||||
tasksA, err := taskStore.ListBySession(ctxA, rootAID, sessionID)
|
||||
if err != nil {
|
||||
t.Fatalf("ListBySession root A: %v", err)
|
||||
}
|
||||
if len(tasksA) != 1 || tasksA[0].ID != taskA {
|
||||
t.Fatalf("ListBySession root A = %#v, want only self-clone %s", tasksA, taskA)
|
||||
}
|
||||
tasksB, err := taskStore.ListBySession(ctxA, rootBID, sessionID)
|
||||
if err != nil {
|
||||
t.Fatalf("ListBySession root B: %v", err)
|
||||
}
|
||||
if len(tasksB) != 1 || tasksB[0].ID != taskB {
|
||||
t.Fatalf("ListBySession root B = %#v, want only %s", tasksB, taskB)
|
||||
}
|
||||
|
||||
if _, err := taskStore.Get(context.Background(), rootAID, taskA); err == nil {
|
||||
t.Fatal("Get without tenant context returned nil error")
|
||||
}
|
||||
if _, err := taskStore.Get(ctxA, uuid.Nil, taskA); !errors.Is(err, store.ErrSubagentRootAgentIDRequired) {
|
||||
t.Fatalf("Get empty root error = %v, want %v", err, store.ErrSubagentRootAgentIDRequired)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPGSubagentTaskStoreRejectsRecreatedAgentWithSameKey(t *testing.T) {
|
||||
db := hooksTestDB(t)
|
||||
tenantID, oldRootAgentID := seedTenantAndAgent(t, db)
|
||||
ctx := tenantScopedCtx(tenantID)
|
||||
taskStore := NewPGSubagentTaskStore(db)
|
||||
|
||||
const rootAgentKey = "recreated-root"
|
||||
if _, err := db.Exec(
|
||||
`UPDATE agents SET agent_key = $1 WHERE id = $2`,
|
||||
rootAgentKey, oldRootAgentID,
|
||||
); err != nil {
|
||||
t.Fatalf("rename original root agent: %v", err)
|
||||
}
|
||||
taskID := createPGSubagentTask(
|
||||
t, taskStore, ctx, oldRootAgentID, rootAgentKey, "recreated-root-session", "queued",
|
||||
)
|
||||
|
||||
if _, err := db.Exec(`DELETE FROM agents WHERE id = $1`, oldRootAgentID); err != nil {
|
||||
t.Fatalf("delete original root agent: %v", err)
|
||||
}
|
||||
newRootAgentID := uuid.Must(uuid.NewV7())
|
||||
if _, err := db.Exec(
|
||||
`INSERT INTO agents (id, tenant_id, agent_key, agent_type, status, provider, model, owner_id)
|
||||
VALUES ($1,$2,$3,'predefined','active','test','test-model','owner')`,
|
||||
newRootAgentID, tenantID, rootAgentKey,
|
||||
); err != nil {
|
||||
t.Fatalf("recreate root agent: %v", err)
|
||||
}
|
||||
|
||||
got, err := taskStore.Get(ctx, newRootAgentID, taskID)
|
||||
if err != nil {
|
||||
t.Fatalf("Get with recreated root agent: %v", err)
|
||||
}
|
||||
if got != nil {
|
||||
t.Fatalf("Get with recreated root agent returned old task %s", got.ID)
|
||||
}
|
||||
if err := taskStore.UpdateStatus(ctx, newRootAgentID, taskID, "completed", nil, 1, 2, 3); !errors.Is(err, store.ErrSubagentTaskNotFound) {
|
||||
t.Fatalf("UpdateStatus with recreated root agent error = %v, want scoped not found", err)
|
||||
}
|
||||
var status string
|
||||
var rootAgentID uuid.NullUUID
|
||||
if err := db.QueryRow(
|
||||
`SELECT status, root_agent_id FROM subagent_tasks WHERE id = $1`, taskID,
|
||||
).Scan(&status, &rootAgentID); err != nil {
|
||||
t.Fatalf("read preserved old task: %v", err)
|
||||
}
|
||||
if status != "queued" || rootAgentID.Valid {
|
||||
t.Fatalf("old task after recreate: status=%q root_agent_id=%s, want queued/NULL", status, rootAgentID.UUID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPGSubagentTaskStoreCompletedAtOnlyForTerminalStatus(t *testing.T) {
|
||||
db := hooksTestDB(t)
|
||||
tenantID, rootAgentID := seedTenantAndAgent(t, db)
|
||||
ctx := tenantScopedCtx(tenantID)
|
||||
taskStore := NewPGSubagentTaskStore(db)
|
||||
|
||||
tests := []struct {
|
||||
status string
|
||||
terminal bool
|
||||
}{
|
||||
{status: "new"},
|
||||
{status: "queued"},
|
||||
{status: "running"},
|
||||
{status: "waiting_child"},
|
||||
{status: "completed", terminal: true},
|
||||
{status: "failed", terminal: true},
|
||||
{status: "cancelled", terminal: true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.status, func(t *testing.T) {
|
||||
rootAgentKey := "root-" + tt.status
|
||||
id := createPGSubagentTask(t, taskStore, ctx, rootAgentID, rootAgentKey, "session-"+tt.status, "queued")
|
||||
if err := taskStore.UpdateStatus(ctx, rootAgentID, id, tt.status, nil, 0, 0, 0); err != nil {
|
||||
t.Fatalf("UpdateStatus(%q): %v", tt.status, err)
|
||||
}
|
||||
got, err := taskStore.Get(ctx, rootAgentID, id)
|
||||
if err != nil {
|
||||
t.Fatalf("Get(%q): %v", tt.status, err)
|
||||
}
|
||||
if (got.CompletedAt != nil) != tt.terminal {
|
||||
t.Fatalf("status %q completed_at = %v, terminal=%v", tt.status, got.CompletedAt, tt.terminal)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPGSubagentTaskStoreRecoverInterrupted(t *testing.T) {
|
||||
db := hooksTestDB(t)
|
||||
// Recovery is intentionally process-global. Reset its shared table so rows
|
||||
// left by other store tests cannot affect the exact recovery count.
|
||||
if _, err := db.Exec(`DELETE FROM subagent_tasks`); err != nil {
|
||||
t.Fatalf("reset subagent tasks before global recovery: %v", err)
|
||||
}
|
||||
tenantA, rootAID := seedTenantAndAgent(t, db)
|
||||
tenantB, rootBID := seedTenantAndAgent(t, db)
|
||||
ctxA := tenantScopedCtx(tenantA)
|
||||
ctxB := tenantScopedCtx(tenantB)
|
||||
taskStore := NewPGSubagentTaskStore(db)
|
||||
|
||||
queuedID := createPGSubagentTask(t, taskStore, ctxA, rootAID, "root-a", "queued", "queued")
|
||||
if _, err := db.Exec(
|
||||
`UPDATE subagent_tasks SET completed_at = $1 WHERE id = $2`,
|
||||
time.Now().UTC().Add(-time.Hour),
|
||||
queuedID,
|
||||
); err != nil {
|
||||
t.Fatalf("seed malformed queued completed_at: %v", err)
|
||||
}
|
||||
runningID := createPGSubagentTask(t, taskStore, ctxA, rootAID, "root-a", "running", "running")
|
||||
waitingID := createPGSubagentTask(
|
||||
t, taskStore, ctxB, rootBID, "root-b", "waiting", "waiting_child",
|
||||
)
|
||||
if err := taskStore.UpdateMetadata(ctxB, rootBID, waitingID, map[string]any{
|
||||
"completion_kind": "delegate",
|
||||
"completion_media": []map[string]any{{
|
||||
"path": ".delegations/completed-before-crash/report.pdf",
|
||||
"mime_type": "application/pdf",
|
||||
}},
|
||||
}); err != nil {
|
||||
t.Fatalf("record published artifact metadata: %v", err)
|
||||
}
|
||||
completedID := createPGSubagentTask(t, taskStore, ctxB, rootBID, "root-b", "completed", "queued")
|
||||
completedResult := "already completed"
|
||||
if err := taskStore.UpdateStatus(
|
||||
ctxB, rootBID, completedID, "completed", &completedResult, 1, 2, 3,
|
||||
); err != nil {
|
||||
t.Fatalf("complete terminal task: %v", err)
|
||||
}
|
||||
|
||||
if _, err := taskStore.RecoverInterrupted(ctxA); err == nil {
|
||||
t.Fatal("RecoverInterrupted accepted tenant-scoped context")
|
||||
}
|
||||
recoveryCtx := store.WithTenantID(context.Background(), store.MasterTenantID)
|
||||
recovered, err := taskStore.RecoverInterrupted(recoveryCtx)
|
||||
if err != nil {
|
||||
t.Fatalf("RecoverInterrupted: %v", err)
|
||||
}
|
||||
if recovered != 3 {
|
||||
t.Fatalf("RecoverInterrupted recovered %d tasks, want 3", recovered)
|
||||
}
|
||||
|
||||
for _, item := range []struct {
|
||||
ctx context.Context
|
||||
root uuid.UUID
|
||||
id uuid.UUID
|
||||
}{
|
||||
{ctx: ctxA, root: rootAID, id: queuedID},
|
||||
{ctx: ctxA, root: rootAID, id: runningID},
|
||||
{ctx: ctxB, root: rootBID, id: waitingID},
|
||||
} {
|
||||
got, getErr := taskStore.Get(item.ctx, item.root, item.id)
|
||||
if getErr != nil {
|
||||
t.Fatalf("Get recovered task %s: %v", item.id, getErr)
|
||||
}
|
||||
if got.Status != "failed" || got.CompletedAt == nil || got.Result == nil ||
|
||||
!strings.Contains(*got.Result, "gateway stopped") {
|
||||
t.Fatalf("recovered task %s = %#v", item.id, got)
|
||||
}
|
||||
if item.id == waitingID {
|
||||
if got.Metadata["completion_kind"] != "delegate" ||
|
||||
got.Metadata["completion_media"] == nil {
|
||||
t.Fatalf("published artifact metadata was lost: %#v", got.Metadata)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
completed, err := taskStore.Get(ctxB, rootBID, completedID)
|
||||
if err != nil {
|
||||
t.Fatalf("Get completed task: %v", err)
|
||||
}
|
||||
if completed.Status != "completed" || completed.Result == nil ||
|
||||
*completed.Result != completedResult {
|
||||
t.Fatalf("completed task changed during recovery: %#v", completed)
|
||||
}
|
||||
|
||||
recovered, err = taskStore.RecoverInterrupted(recoveryCtx)
|
||||
if err != nil {
|
||||
t.Fatalf("RecoverInterrupted second pass: %v", err)
|
||||
}
|
||||
if recovered != 0 {
|
||||
t.Fatalf("RecoverInterrupted second pass recovered %d tasks, want 0", recovered)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPGSubagentTaskStoreArchiveIsScopedAndBounded(t *testing.T) {
|
||||
db := hooksTestDB(t)
|
||||
tenantA, rootAID := seedTenantAndAgent(t, db)
|
||||
tenantB, tenantBRootID := seedTenantAndAgent(t, db)
|
||||
ctxA := tenantScopedCtx(tenantA)
|
||||
ctxB := tenantScopedCtx(tenantB)
|
||||
taskStore := NewPGSubagentTaskStore(db)
|
||||
|
||||
const (
|
||||
rootA = "root-a"
|
||||
rootB = "root-b"
|
||||
)
|
||||
rootBID := uuid.Must(uuid.NewV7())
|
||||
if _, err := db.Exec(
|
||||
`INSERT INTO agents (id, tenant_id, agent_key, agent_type, status, provider, model, owner_id)
|
||||
VALUES ($1,$2,$3,'predefined','active','test','test-model','owner')`,
|
||||
rootBID, tenantA, rootB,
|
||||
); err != nil {
|
||||
t.Fatalf("seed second root agent: %v", err)
|
||||
}
|
||||
var rootATasks []uuid.UUID
|
||||
for i := range 3 {
|
||||
rootATasks = append(rootATasks, createPGSubagentTask(
|
||||
t, taskStore, ctxA, rootAID, rootA, fmt.Sprintf("session-a-%d", i), "queued",
|
||||
))
|
||||
}
|
||||
rootBTask := createPGSubagentTask(t, taskStore, ctxA, rootBID, rootB, "session-b", "queued")
|
||||
tenantBTask := createPGSubagentTask(t, taskStore, ctxB, tenantBRootID, rootA, "session-other-tenant", "queued")
|
||||
queuedTask := createPGSubagentTask(t, taskStore, ctxA, rootAID, rootA, "session-queued", "queued")
|
||||
|
||||
for _, item := range []struct {
|
||||
ctx context.Context
|
||||
root uuid.UUID
|
||||
id uuid.UUID
|
||||
}{
|
||||
{ctx: ctxA, root: rootAID, id: rootATasks[0]},
|
||||
{ctx: ctxA, root: rootAID, id: rootATasks[1]},
|
||||
{ctx: ctxA, root: rootAID, id: rootATasks[2]},
|
||||
{ctx: ctxA, root: rootBID, id: rootBTask},
|
||||
{ctx: ctxB, root: tenantBRootID, id: tenantBTask},
|
||||
} {
|
||||
if err := taskStore.UpdateStatus(item.ctx, item.root, item.id, "completed", nil, 0, 0, 0); err != nil {
|
||||
t.Fatalf("UpdateStatus(%s): %v", item.id, err)
|
||||
}
|
||||
}
|
||||
|
||||
oldTime := time.Now().UTC().Add(-2 * time.Hour)
|
||||
for _, id := range append(append([]uuid.UUID{}, rootATasks...), rootBTask, tenantBTask) {
|
||||
if _, err := db.Exec(`UPDATE subagent_tasks SET completed_at = $1 WHERE id = $2`, oldTime, id); err != nil {
|
||||
t.Fatalf("backdate terminal task %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
// A non-terminal row must remain unarchived even if malformed legacy data
|
||||
// happens to carry a completed_at value.
|
||||
if _, err := db.Exec(`UPDATE subagent_tasks SET completed_at = $1 WHERE id = $2`, oldTime, queuedTask); err != nil {
|
||||
t.Fatalf("backdate queued task: %v", err)
|
||||
}
|
||||
|
||||
archived, err := taskStore.Archive(ctxA, rootAID, time.Hour, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("Archive first batch: %v", err)
|
||||
}
|
||||
if archived != 2 {
|
||||
t.Fatalf("Archive first batch affected %d rows, want 2", archived)
|
||||
}
|
||||
|
||||
assertPGArchivedCount(t, db, tenantA, rootAID, 2)
|
||||
assertPGArchivedCount(t, db, tenantA, rootBID, 0)
|
||||
assertPGArchivedCount(t, db, tenantB, tenantBRootID, 0)
|
||||
|
||||
archived, err = taskStore.Archive(ctxA, rootAID, time.Hour, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("Archive second batch: %v", err)
|
||||
}
|
||||
if archived != 1 {
|
||||
t.Fatalf("Archive second batch affected %d rows, want 1", archived)
|
||||
}
|
||||
assertPGArchivedCount(t, db, tenantA, rootAID, 3)
|
||||
|
||||
var queuedArchived sql.NullTime
|
||||
if err := db.QueryRow(`SELECT archived_at FROM subagent_tasks WHERE id = $1`, queuedTask).Scan(&queuedArchived); err != nil {
|
||||
t.Fatalf("read queued archived_at: %v", err)
|
||||
}
|
||||
if queuedArchived.Valid {
|
||||
t.Fatalf("queued task archived_at = %s, want NULL", queuedArchived.Time)
|
||||
}
|
||||
}
|
||||
|
||||
func assertPGArchivedCount(
|
||||
t *testing.T, db *sql.DB, tenantID, rootAgentID uuid.UUID, want int,
|
||||
) {
|
||||
t.Helper()
|
||||
var got int
|
||||
if err := db.QueryRow(
|
||||
`SELECT COUNT(*) FROM subagent_tasks
|
||||
WHERE tenant_id = $1 AND root_agent_id = $2 AND archived_at IS NOT NULL`,
|
||||
tenantID, rootAgentID,
|
||||
).Scan(&got); err != nil {
|
||||
t.Fatalf("count archived tasks for %s/%s: %v", tenantID, rootAgentID, err)
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("archived tasks for %s/%s = %d, want %d", tenantID, rootAgentID, got, want)
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -207,6 +208,7 @@ func (s *SQLiteAgentLinkStore) GetLinkBetween(ctx context.Context, fromAgentID,
|
||||
tenantFilter = " AND tenant_id = ?"
|
||||
args = append(args, tenantID)
|
||||
}
|
||||
args = append(args, fromAgentID)
|
||||
row := s.db.QueryRowContext(ctx,
|
||||
`SELECT `+linkSelectCols+`
|
||||
FROM agent_links WHERE status = 'active'
|
||||
@@ -214,11 +216,16 @@ func (s *SQLiteAgentLinkStore) GetLinkBetween(ctx context.Context, fromAgentID,
|
||||
(source_agent_id = ? AND target_agent_id = ? AND direction IN ('outbound', 'bidirectional'))
|
||||
OR
|
||||
(source_agent_id = ? AND target_agent_id = ? AND direction IN ('inbound', 'bidirectional'))
|
||||
)`+tenantFilter+` LIMIT 1`, args...)
|
||||
)`+tenantFilter+`
|
||||
ORDER BY CASE WHEN source_agent_id = ? THEN 0 ELSE 1 END, created_at, id
|
||||
LIMIT 1`, args...)
|
||||
d, err := scanLinkRow(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
//go:build sqlite || sqliteonly
|
||||
|
||||
package sqlitestore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
)
|
||||
|
||||
func seedSQLiteLinkAgent(t *testing.T, db *sql.DB, tenantID uuid.UUID, key string) uuid.UUID {
|
||||
t.Helper()
|
||||
id := uuid.New()
|
||||
_, err := db.Exec(
|
||||
`INSERT INTO agents (id, tenant_id, agent_key, agent_type, status, provider, model, owner_id)
|
||||
VALUES (?,?,?,'predefined','active','test','test-model','owner')`,
|
||||
id.String(), tenantID.String(), key)
|
||||
if err != nil {
|
||||
t.Fatalf("seed agent %q: %v", key, err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func newSQLiteLinkFixture(t *testing.T) (*SQLiteAgentLinkStore, *sql.DB, context.Context, uuid.UUID, uuid.UUID, uuid.UUID) {
|
||||
t.Helper()
|
||||
db := newHookTestDB(t)
|
||||
tenantID, agentA := seedHookTenantAgent(t, db)
|
||||
if _, err := db.Exec(`UPDATE agents SET agent_key = 'agent-a' WHERE id = ?`, agentA.String()); err != nil {
|
||||
t.Fatalf("rename agent A: %v", err)
|
||||
}
|
||||
agentB := seedSQLiteLinkAgent(t, db, tenantID, "agent-b")
|
||||
agentC := seedSQLiteLinkAgent(t, db, tenantID, "agent-c")
|
||||
ctx := store.WithTenantID(context.Background(), tenantID)
|
||||
return NewSQLiteAgentLinkStore(db), db, ctx, agentA, agentB, agentC
|
||||
}
|
||||
|
||||
func sqliteLink(source, target uuid.UUID, direction string) *store.AgentLinkData {
|
||||
return &store.AgentLinkData{
|
||||
SourceAgentID: source,
|
||||
TargetAgentID: target,
|
||||
Direction: direction,
|
||||
Status: store.LinkStatusActive,
|
||||
MaxConcurrent: 1,
|
||||
CreatedBy: "test",
|
||||
}
|
||||
}
|
||||
|
||||
func targetKeys(t *testing.T, linkStore *SQLiteAgentLinkStore, ctx context.Context, from uuid.UUID) []string {
|
||||
t.Helper()
|
||||
targets, err := linkStore.DelegateTargets(ctx, from)
|
||||
if err != nil {
|
||||
t.Fatalf("DelegateTargets(%s): %v", from, err)
|
||||
}
|
||||
keys := make([]string, len(targets))
|
||||
for i := range targets {
|
||||
keys[i] = targets[i].TargetAgentKey
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
func TestSQLiteAgentLinkStoreDelegateTargetsDirections(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
direction string
|
||||
wantFromA []string
|
||||
wantFromB []string
|
||||
}{
|
||||
{
|
||||
name: "outbound",
|
||||
direction: store.LinkDirectionOutbound,
|
||||
wantFromA: []string{"agent-b"},
|
||||
},
|
||||
{
|
||||
name: "inbound",
|
||||
direction: store.LinkDirectionInbound,
|
||||
wantFromB: []string{"agent-a"},
|
||||
},
|
||||
{
|
||||
name: "bidirectional",
|
||||
direction: store.LinkDirectionBidirectional,
|
||||
wantFromA: []string{"agent-b"},
|
||||
wantFromB: []string{"agent-a"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
linkStore, _, ctx, agentA, agentB, _ := newSQLiteLinkFixture(t)
|
||||
if err := linkStore.CreateLink(ctx, sqliteLink(agentA, agentB, tt.direction)); err != nil {
|
||||
t.Fatalf("CreateLink: %v", err)
|
||||
}
|
||||
|
||||
if got := targetKeys(t, linkStore, ctx, agentA); !slicesEqual(got, tt.wantFromA) {
|
||||
t.Fatalf("targets from A = %v, want %v", got, tt.wantFromA)
|
||||
}
|
||||
if got := targetKeys(t, linkStore, ctx, agentB); !slicesEqual(got, tt.wantFromB) {
|
||||
t.Fatalf("targets from B = %v, want %v", got, tt.wantFromB)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteAgentLinkStoreDelegateTargetsRefreshesAfterMutations(t *testing.T) {
|
||||
linkStore, db, ctx, agentA, agentB, _ := newSQLiteLinkFixture(t)
|
||||
link := sqliteLink(agentA, agentB, store.LinkDirectionOutbound)
|
||||
if err := linkStore.CreateLink(ctx, link); err != nil {
|
||||
t.Fatalf("CreateLink: %v", err)
|
||||
}
|
||||
if got := targetKeys(t, linkStore, ctx, agentA); !slicesEqual(got, []string{"agent-b"}) {
|
||||
t.Fatalf("initial targets = %v", got)
|
||||
}
|
||||
|
||||
if err := linkStore.UpdateLink(ctx, link.ID, map[string]any{"status": store.LinkStatusDisabled}); err != nil {
|
||||
t.Fatalf("disable link: %v", err)
|
||||
}
|
||||
if got := targetKeys(t, linkStore, ctx, agentA); len(got) != 0 {
|
||||
t.Fatalf("targets after disable = %v, want none", got)
|
||||
}
|
||||
|
||||
if err := linkStore.UpdateLink(ctx, link.ID, map[string]any{
|
||||
"status": store.LinkStatusActive,
|
||||
"direction": store.LinkDirectionInbound,
|
||||
}); err != nil {
|
||||
t.Fatalf("reverse link direction: %v", err)
|
||||
}
|
||||
if got := targetKeys(t, linkStore, ctx, agentA); len(got) != 0 {
|
||||
t.Fatalf("targets from A after inbound update = %v, want none", got)
|
||||
}
|
||||
if got := targetKeys(t, linkStore, ctx, agentB); !slicesEqual(got, []string{"agent-a"}) {
|
||||
t.Fatalf("targets from B after inbound update = %v", got)
|
||||
}
|
||||
|
||||
if _, err := db.Exec(`UPDATE agents SET status = 'inactive' WHERE id = ?`, agentA.String()); err != nil {
|
||||
t.Fatalf("deactivate effective target: %v", err)
|
||||
}
|
||||
if got := targetKeys(t, linkStore, ctx, agentB); len(got) != 0 {
|
||||
t.Fatalf("targets with inactive effective target = %v, want none", got)
|
||||
}
|
||||
|
||||
if _, err := db.Exec(`UPDATE agents SET status = 'active' WHERE id = ?`, agentA.String()); err != nil {
|
||||
t.Fatalf("reactivate target: %v", err)
|
||||
}
|
||||
if err := linkStore.DeleteLink(ctx, link.ID); err != nil {
|
||||
t.Fatalf("DeleteLink: %v", err)
|
||||
}
|
||||
if got := targetKeys(t, linkStore, ctx, agentB); len(got) != 0 {
|
||||
t.Fatalf("targets after delete = %v, want none", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteAgentLinkStoreDelegateTargetsTenantScoped(t *testing.T) {
|
||||
linkStore, _, ctx, agentA, agentB, _ := newSQLiteLinkFixture(t)
|
||||
if err := linkStore.CreateLink(ctx, sqliteLink(agentA, agentB, store.LinkDirectionOutbound)); err != nil {
|
||||
t.Fatalf("CreateLink: %v", err)
|
||||
}
|
||||
|
||||
otherTenantCtx := store.WithTenantID(context.Background(), uuid.New())
|
||||
if got := targetKeys(t, linkStore, otherTenantCtx, agentA); len(got) != 0 {
|
||||
t.Fatalf("cross-tenant targets = %v, want none", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteAgentLinkStoreGetLinkBetweenNotFound(t *testing.T) {
|
||||
linkStore, _, ctx, agentA, agentB, _ := newSQLiteLinkFixture(t)
|
||||
|
||||
link, err := linkStore.GetLinkBetween(ctx, agentA, agentB)
|
||||
if err != nil {
|
||||
t.Fatalf("GetLinkBetween: %v", err)
|
||||
}
|
||||
if link != nil {
|
||||
t.Fatalf("GetLinkBetween = %#v, want nil", link)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteAgentLinkStoreGetLinkBetweenPrefersDelegatorAuthoredRow(t *testing.T) {
|
||||
linkStore, _, ctx, agentA, agentB, _ := newSQLiteLinkFixture(t)
|
||||
|
||||
reverse := sqliteLink(agentB, agentA, store.LinkDirectionInbound)
|
||||
if err := linkStore.CreateLink(ctx, reverse); err != nil {
|
||||
t.Fatalf("CreateLink(reverse): %v", err)
|
||||
}
|
||||
direct := sqliteLink(agentA, agentB, store.LinkDirectionOutbound)
|
||||
if err := linkStore.CreateLink(ctx, direct); err != nil {
|
||||
t.Fatalf("CreateLink(direct): %v", err)
|
||||
}
|
||||
|
||||
link, err := linkStore.GetLinkBetween(ctx, agentA, agentB)
|
||||
if err != nil {
|
||||
t.Fatalf("GetLinkBetween: %v", err)
|
||||
}
|
||||
if link == nil {
|
||||
t.Fatal("GetLinkBetween returned nil")
|
||||
}
|
||||
if link.ID != direct.ID {
|
||||
t.Fatalf("GetLinkBetween chose %s, want delegator-authored %s", link.ID, direct.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteAgentLinkStoreGetLinkBetweenPropagatesDatabaseError(t *testing.T) {
|
||||
linkStore, db, ctx, agentA, agentB, _ := newSQLiteLinkFixture(t)
|
||||
if err := db.Close(); err != nil {
|
||||
t.Fatalf("close fixture DB: %v", err)
|
||||
}
|
||||
|
||||
if _, err := linkStore.GetLinkBetween(ctx, agentA, agentB); err == nil {
|
||||
t.Fatal("GetLinkBetween error = nil after database close")
|
||||
}
|
||||
}
|
||||
|
||||
func slicesEqual(got, want []string) bool {
|
||||
if len(got) != len(want) {
|
||||
return false
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != want[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -34,6 +34,7 @@ func NewSQLiteStores(cfg store.StoreConfig) (*store.Stores, error) {
|
||||
} else {
|
||||
slog.Warn("securecli: encryption key empty, store disabled")
|
||||
}
|
||||
subagentTasks := NewSQLiteSubagentTaskStore(db)
|
||||
|
||||
sqliteStores := &store.Stores{
|
||||
DB: db,
|
||||
@@ -67,7 +68,8 @@ func NewSQLiteStores(cfg store.StoreConfig) (*store.Stores, error) {
|
||||
ConfigPermissions: NewSQLiteConfigPermissionStore(db),
|
||||
BrowserCookies: NewSQLiteBrowserCookieStore(db, cfg.EncryptionKey),
|
||||
Memory: NewSQLiteMemoryStore(db),
|
||||
SubagentTasks: NewSQLiteSubagentTaskStore(db),
|
||||
SubagentTasks: subagentTasks,
|
||||
SubagentTaskRecovery: subagentTasks,
|
||||
AgentLinks: NewSQLiteAgentLinkStore(db),
|
||||
SecureCLI: secureCLI,
|
||||
SecureCLIGrants: NewSQLiteSecureCLIAgentGrantStore(db, cfg.EncryptionKey),
|
||||
|
||||
@@ -16,7 +16,7 @@ var schemaSQL string
|
||||
|
||||
// SchemaVersion is the current SQLite schema version.
|
||||
// Bump this when adding new migration steps below.
|
||||
const SchemaVersion = 58
|
||||
const SchemaVersion = 59
|
||||
|
||||
// migrations maps version → SQL to apply when upgrading FROM that version.
|
||||
// schema.sql always represents the LATEST full schema (for fresh DBs).
|
||||
@@ -29,7 +29,78 @@ const SchemaVersion = 58
|
||||
// }
|
||||
//
|
||||
// Then bump SchemaVersion to 2.
|
||||
const sqliteSubagentRootAgentScopeMigrationBody = `UPDATE subagent_tasks
|
||||
SET root_agent_id = json_extract(metadata, '$.root_agent_id')
|
||||
WHERE root_agent_id IS NULL
|
||||
AND json_valid(metadata)
|
||||
AND typeof(json_extract(metadata, '$.root_agent_id')) = 'text'
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM agents
|
||||
WHERE agents.tenant_id = subagent_tasks.tenant_id
|
||||
AND agents.id = json_extract(subagent_tasks.metadata, '$.root_agent_id')
|
||||
);
|
||||
UPDATE subagent_tasks
|
||||
SET root_agent_id = (
|
||||
SELECT MIN(agents.id)
|
||||
FROM agents
|
||||
WHERE agents.tenant_id = subagent_tasks.tenant_id
|
||||
AND agents.agent_key = subagent_tasks.parent_agent_key
|
||||
AND agents.created_at < subagent_tasks.created_at
|
||||
)
|
||||
WHERE root_agent_id IS NULL
|
||||
AND CASE
|
||||
WHEN json_valid(metadata) THEN json_type(metadata, '$.root_agent_id') IS NULL
|
||||
ELSE 1
|
||||
END
|
||||
AND (
|
||||
SELECT COUNT(*)
|
||||
FROM agents
|
||||
WHERE agents.tenant_id = subagent_tasks.tenant_id
|
||||
AND agents.agent_key = subagent_tasks.parent_agent_key
|
||||
AND agents.created_at < subagent_tasks.created_at
|
||||
) = 1;
|
||||
CREATE INDEX IF NOT EXISTS idx_subagent_tasks_root_status
|
||||
ON subagent_tasks(tenant_id, root_agent_id, status, created_at DESC)
|
||||
WHERE root_agent_id IS NOT NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_subagent_tasks_root_session
|
||||
ON subagent_tasks(tenant_id, root_agent_id, session_key, created_at DESC)
|
||||
WHERE root_agent_id IS NOT NULL AND session_key IS NOT NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_subagent_tasks_root_archive
|
||||
ON subagent_tasks(tenant_id, root_agent_id, completed_at, id)
|
||||
WHERE root_agent_id IS NOT NULL
|
||||
AND status IN ('completed', 'failed', 'cancelled')
|
||||
AND archived_at IS NULL;
|
||||
CREATE TRIGGER IF NOT EXISTS trg_subagent_tasks_root_tenant_insert
|
||||
BEFORE INSERT ON subagent_tasks
|
||||
WHEN NEW.root_agent_id IS NOT NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM agents
|
||||
WHERE agents.id = NEW.root_agent_id
|
||||
AND agents.tenant_id = NEW.tenant_id
|
||||
)
|
||||
BEGIN
|
||||
SELECT RAISE(ABORT, 'subagent root agent belongs to another tenant');
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS trg_subagent_tasks_root_tenant_update
|
||||
BEFORE UPDATE OF root_agent_id, tenant_id ON subagent_tasks
|
||||
WHEN NEW.root_agent_id IS NOT NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM agents
|
||||
WHERE agents.id = NEW.root_agent_id
|
||||
AND agents.tenant_id = NEW.tenant_id
|
||||
)
|
||||
BEGIN
|
||||
SELECT RAISE(ABORT, 'subagent root agent belongs to another tenant');
|
||||
END;`
|
||||
|
||||
var migrations = map[int]string{
|
||||
// Version 58 → 59: scope persisted subagent tasks by immutable root-agent UUID.
|
||||
// Metadata is authoritative; key fallback is allowed only for one matching
|
||||
// agent that predates the task. Unmatched rows remain inaccessible.
|
||||
58: `ALTER TABLE subagent_tasks
|
||||
ADD COLUMN root_agent_id TEXT REFERENCES agents(id) ON DELETE SET NULL;
|
||||
` + sqliteSubagentRootAgentScopeMigrationBody,
|
||||
// Version 57 → 58: restore custom skills previously converted by the bundled skill seeder.
|
||||
57: `UPDATE skills
|
||||
SET is_system = 0,
|
||||
@@ -1521,6 +1592,12 @@ func EnsureSchema(db *sql.DB) error {
|
||||
return fmt.Errorf("inspect channel pending message parent column: %w", err)
|
||||
}
|
||||
}
|
||||
if v == 58 {
|
||||
patch, err = sqliteSubagentRootAgentMigrationPatch(db)
|
||||
if err != nil {
|
||||
return fmt.Errorf("inspect subagent task root-agent column: %w", err)
|
||||
}
|
||||
}
|
||||
// Migrations that rebuild a table referenced by another table's FK
|
||||
// require foreign_keys=OFF per SQLite altertable §7. The pragma is
|
||||
// a no-op inside a transaction, so toggle it around BEGIN/COMMIT.
|
||||
@@ -1681,6 +1758,19 @@ func sqlitePendingMessageParentMigrationPatch(db *sql.DB) (string, error) {
|
||||
return patch, nil
|
||||
}
|
||||
|
||||
func sqliteSubagentRootAgentMigrationPatch(db *sql.DB) (string, error) {
|
||||
hasColumn, err := sqliteColumnExists(db, "subagent_tasks", "root_agent_id")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
patch := ""
|
||||
if !hasColumn {
|
||||
patch += "ALTER TABLE subagent_tasks ADD COLUMN root_agent_id TEXT REFERENCES agents(id) ON DELETE SET NULL;\n"
|
||||
}
|
||||
patch += sqliteSubagentRootAgentScopeMigrationBody
|
||||
return patch, nil
|
||||
}
|
||||
|
||||
func sqliteColumnExists(db *sql.DB, tableName, columnName string) (bool, error) {
|
||||
rows, err := db.Query("PRAGMA table_info(" + tableName + ")")
|
||||
if err != nil {
|
||||
|
||||
@@ -1678,6 +1678,7 @@ CREATE INDEX IF NOT EXISTS idx_system_configs_tenant ON system_configs(tenant_id
|
||||
CREATE TABLE IF NOT EXISTS subagent_tasks (
|
||||
id TEXT PRIMARY KEY,
|
||||
tenant_id TEXT NOT NULL REFERENCES tenants(id) ON DELETE CASCADE,
|
||||
root_agent_id TEXT REFERENCES agents(id) ON DELETE SET NULL,
|
||||
parent_agent_key VARCHAR(255) NOT NULL,
|
||||
session_key VARCHAR(500),
|
||||
subject VARCHAR(255) NOT NULL,
|
||||
@@ -1706,6 +1707,39 @@ CREATE TABLE IF NOT EXISTS subagent_tasks (
|
||||
CREATE INDEX IF NOT EXISTS idx_subagent_tasks_parent_status ON subagent_tasks(tenant_id, parent_agent_key, status);
|
||||
CREATE INDEX IF NOT EXISTS idx_subagent_tasks_session ON subagent_tasks(session_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_subagent_tasks_created ON subagent_tasks(tenant_id, created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_subagent_tasks_root_status
|
||||
ON subagent_tasks(tenant_id, root_agent_id, status, created_at DESC)
|
||||
WHERE root_agent_id IS NOT NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_subagent_tasks_root_session
|
||||
ON subagent_tasks(tenant_id, root_agent_id, session_key, created_at DESC)
|
||||
WHERE root_agent_id IS NOT NULL AND session_key IS NOT NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_subagent_tasks_root_archive
|
||||
ON subagent_tasks(tenant_id, root_agent_id, completed_at, id)
|
||||
WHERE root_agent_id IS NOT NULL
|
||||
AND status IN ('completed', 'failed', 'cancelled')
|
||||
AND archived_at IS NULL;
|
||||
CREATE TRIGGER IF NOT EXISTS trg_subagent_tasks_root_tenant_insert
|
||||
BEFORE INSERT ON subagent_tasks
|
||||
WHEN NEW.root_agent_id IS NOT NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM agents
|
||||
WHERE agents.id = NEW.root_agent_id
|
||||
AND agents.tenant_id = NEW.tenant_id
|
||||
)
|
||||
BEGIN
|
||||
SELECT RAISE(ABORT, 'subagent root agent belongs to another tenant');
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS trg_subagent_tasks_root_tenant_update
|
||||
BEFORE UPDATE OF root_agent_id, tenant_id ON subagent_tasks
|
||||
WHEN NEW.root_agent_id IS NOT NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM agents
|
||||
WHERE agents.id = NEW.root_agent_id
|
||||
AND agents.tenant_id = NEW.tenant_id
|
||||
)
|
||||
BEGIN
|
||||
SELECT RAISE(ABORT, 'subagent root agent belongs to another tenant');
|
||||
END;
|
||||
|
||||
-- ============================================================
|
||||
-- Table: episodic_summaries (V3 Tier 2 memory)
|
||||
|
||||
@@ -22,16 +22,18 @@ type SQLiteSubagentTaskStore struct {
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
var _ store.SubagentTaskRecoveryStore = (*SQLiteSubagentTaskStore)(nil)
|
||||
|
||||
// NewSQLiteSubagentTaskStore creates a new SQLiteSubagentTaskStore.
|
||||
func NewSQLiteSubagentTaskStore(db *sql.DB) *SQLiteSubagentTaskStore {
|
||||
return &SQLiteSubagentTaskStore{db: db}
|
||||
}
|
||||
|
||||
const subagentTaskInsertCols = `tenant_id, parent_agent_key, session_key, subject, description,
|
||||
const subagentTaskInsertCols = `tenant_id, root_agent_id, parent_agent_key, session_key, subject, description,
|
||||
status, result, depth, model, provider, iterations, input_tokens, output_tokens,
|
||||
origin_channel, origin_chat_id, origin_peer_kind, origin_user_id, spawned_by, metadata`
|
||||
|
||||
const subagentTaskSelectCols = `id, tenant_id, parent_agent_key, session_key, subject, description,
|
||||
const subagentTaskSelectCols = `id, tenant_id, root_agent_id, parent_agent_key, session_key, subject, description,
|
||||
status, result, depth, model, provider, iterations, input_tokens, output_tokens,
|
||||
origin_channel, origin_chat_id, origin_peer_kind, origin_user_id, spawned_by,
|
||||
completed_at, archived_at, COALESCE(metadata, '{}'), created_at, updated_at`
|
||||
@@ -39,6 +41,9 @@ const subagentTaskSelectCols = `id, tenant_id, parent_agent_key, session_key, su
|
||||
// Create persists a new subagent task at spawn time.
|
||||
func (s *SQLiteSubagentTaskStore) Create(ctx context.Context, task *store.SubagentTaskData) error {
|
||||
tid := tenantIDForInsert(ctx)
|
||||
if task.RootAgentID == uuid.Nil {
|
||||
return store.ErrSubagentRootAgentIDRequired
|
||||
}
|
||||
|
||||
metaJSON := []byte("{}")
|
||||
if len(task.Metadata) > 0 {
|
||||
@@ -49,10 +54,10 @@ func (s *SQLiteSubagentTaskStore) Create(ctx context.Context, task *store.Subage
|
||||
|
||||
now := time.Now().UTC().Format(time.RFC3339Nano)
|
||||
q := fmt.Sprintf(`INSERT OR IGNORE INTO subagent_tasks (id, %s, created_at, updated_at)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, subagentTaskInsertCols)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, subagentTaskInsertCols)
|
||||
|
||||
_, err := s.db.ExecContext(ctx, q,
|
||||
task.ID, tid, task.ParentAgentKey, task.SessionKey, task.Subject, task.Description,
|
||||
task.ID, tid, task.RootAgentID, task.ParentAgentKey, task.SessionKey, task.Subject, task.Description,
|
||||
task.Status, task.Result, task.Depth, task.Model, task.Provider,
|
||||
task.Iterations, task.InputTokens, task.OutputTokens,
|
||||
task.OriginChannel, task.OriginChatID, task.OriginPeerKind, task.OriginUserID,
|
||||
@@ -70,7 +75,7 @@ func scanTask(row interface{ Scan(...any) error }) (*store.SubagentTaskData, err
|
||||
var createdAt, updatedAt sqliteTime
|
||||
|
||||
err := row.Scan(
|
||||
&t.ID, &t.TenantID, &t.ParentAgentKey, &t.SessionKey, &t.Subject, &t.Description,
|
||||
&t.ID, &t.TenantID, &t.RootAgentID, &t.ParentAgentKey, &t.SessionKey, &t.Subject, &t.Description,
|
||||
&t.Status, &t.Result, &t.Depth, &t.Model, &t.Provider,
|
||||
&t.Iterations, &t.InputTokens, &t.OutputTokens,
|
||||
&t.OriginChannel, &t.OriginChatID, &t.OriginPeerKind, &t.OriginUserID, &t.SpawnedBy,
|
||||
@@ -95,14 +100,20 @@ func scanTask(row interface{ Scan(...any) error }) (*store.SubagentTaskData, err
|
||||
return &t, nil
|
||||
}
|
||||
|
||||
// Get retrieves a single task by ID (tenant-scoped).
|
||||
func (s *SQLiteSubagentTaskStore) Get(ctx context.Context, id uuid.UUID) (*store.SubagentTaskData, error) {
|
||||
// Get retrieves a task owned by the tenant and immutable root-agent UUID.
|
||||
func (s *SQLiteSubagentTaskStore) Get(
|
||||
ctx context.Context, rootAgentID, id uuid.UUID,
|
||||
) (*store.SubagentTaskData, error) {
|
||||
tid, err := requireTenantID(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
q := fmt.Sprintf(`SELECT %s FROM subagent_tasks WHERE id = ? AND tenant_id = ?`, subagentTaskSelectCols)
|
||||
row := s.db.QueryRowContext(ctx, q, id, tid)
|
||||
if rootAgentID == uuid.Nil {
|
||||
return nil, store.ErrSubagentRootAgentIDRequired
|
||||
}
|
||||
q := fmt.Sprintf(`SELECT %s FROM subagent_tasks
|
||||
WHERE id = ? AND tenant_id = ? AND root_agent_id = ?`, subagentTaskSelectCols)
|
||||
row := s.db.QueryRowContext(ctx, q, id, tid, rootAgentID)
|
||||
t, err := scanTask(row)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
@@ -112,7 +123,7 @@ func (s *SQLiteSubagentTaskStore) Get(ctx context.Context, id uuid.UUID) (*store
|
||||
|
||||
// UpdateStatus updates status, result, iterations, and token counts on completion/failure.
|
||||
func (s *SQLiteSubagentTaskStore) UpdateStatus(
|
||||
ctx context.Context, id uuid.UUID,
|
||||
ctx context.Context, rootAgentID, id uuid.UUID,
|
||||
status string, result *string, iterations int,
|
||||
inputTokens, outputTokens int64,
|
||||
) error {
|
||||
@@ -120,10 +131,13 @@ func (s *SQLiteSubagentTaskStore) UpdateStatus(
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rootAgentID == uuid.Nil {
|
||||
return store.ErrSubagentRootAgentIDRequired
|
||||
}
|
||||
|
||||
now := time.Now().UTC().Format(time.RFC3339Nano)
|
||||
var completedAt *string
|
||||
if status != "running" {
|
||||
if store.IsTerminalSubagentTaskStatus(status) {
|
||||
v := now
|
||||
completedAt = &v
|
||||
}
|
||||
@@ -132,34 +146,49 @@ func (s *SQLiteSubagentTaskStore) UpdateStatus(
|
||||
status = ?, result = ?, iterations = ?,
|
||||
input_tokens = ?, output_tokens = ?,
|
||||
completed_at = ?, updated_at = ?
|
||||
WHERE id = ? AND tenant_id = ?`
|
||||
_, err = s.db.ExecContext(ctx, q,
|
||||
WHERE id = ? AND tenant_id = ? AND root_agent_id = ?`
|
||||
res, err := s.db.ExecContext(ctx, q,
|
||||
status, result, iterations, inputTokens, outputTokens,
|
||||
completedAt, now, id, tid,
|
||||
completedAt, now, id, tid, rootAgentID,
|
||||
)
|
||||
return err
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
affected, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected != 1 {
|
||||
return store.ErrSubagentTaskNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListByParent returns tasks for a parent agent key, optionally filtered by status.
|
||||
// ListByParent returns tasks for a root-agent UUID, optionally filtered by status.
|
||||
func (s *SQLiteSubagentTaskStore) ListByParent(
|
||||
ctx context.Context, parentAgentKey string, statusFilter string,
|
||||
ctx context.Context, rootAgentID uuid.UUID, statusFilter string,
|
||||
) ([]store.SubagentTaskData, error) {
|
||||
tid, err := requireTenantID(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if rootAgentID == uuid.Nil {
|
||||
return nil, store.ErrSubagentRootAgentIDRequired
|
||||
}
|
||||
|
||||
var rows *sql.Rows
|
||||
if statusFilter != "" {
|
||||
q := fmt.Sprintf(`SELECT %s FROM subagent_tasks
|
||||
WHERE tenant_id = ? AND parent_agent_key = ? AND status = ?
|
||||
ORDER BY created_at DESC LIMIT 50`, subagentTaskSelectCols)
|
||||
rows, err = s.db.QueryContext(ctx, q, tid, parentAgentKey, statusFilter)
|
||||
WHERE tenant_id = ? AND root_agent_id = ? AND status = ?
|
||||
AND COALESCE(json_extract(metadata, '$.completion_kind'), 'subagent') <> 'delegate'
|
||||
ORDER BY created_at DESC LIMIT 50`, subagentTaskSelectCols)
|
||||
rows, err = s.db.QueryContext(ctx, q, tid, rootAgentID, statusFilter)
|
||||
} else {
|
||||
q := fmt.Sprintf(`SELECT %s FROM subagent_tasks
|
||||
WHERE tenant_id = ? AND parent_agent_key = ?
|
||||
ORDER BY created_at DESC LIMIT 50`, subagentTaskSelectCols)
|
||||
rows, err = s.db.QueryContext(ctx, q, tid, parentAgentKey)
|
||||
WHERE tenant_id = ? AND root_agent_id = ?
|
||||
AND COALESCE(json_extract(metadata, '$.completion_kind'), 'subagent') <> 'delegate'
|
||||
ORDER BY created_at DESC LIMIT 50`, subagentTaskSelectCols)
|
||||
rows, err = s.db.QueryContext(ctx, q, tid, rootAgentID)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -170,17 +199,21 @@ func (s *SQLiteSubagentTaskStore) ListByParent(
|
||||
|
||||
// ListBySession returns tasks for a specific session key (tenant-scoped).
|
||||
func (s *SQLiteSubagentTaskStore) ListBySession(
|
||||
ctx context.Context, sessionKey string,
|
||||
ctx context.Context, rootAgentID uuid.UUID, sessionKey string,
|
||||
) ([]store.SubagentTaskData, error) {
|
||||
tid, err := requireTenantID(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if rootAgentID == uuid.Nil {
|
||||
return nil, store.ErrSubagentRootAgentIDRequired
|
||||
}
|
||||
|
||||
q := fmt.Sprintf(`SELECT %s FROM subagent_tasks
|
||||
WHERE tenant_id = ? AND session_key = ?
|
||||
WHERE tenant_id = ? AND root_agent_id = ? AND session_key = ?
|
||||
AND COALESCE(json_extract(metadata, '$.completion_kind'), 'subagent') <> 'delegate'
|
||||
ORDER BY created_at DESC LIMIT 50`, subagentTaskSelectCols)
|
||||
rows, err := s.db.QueryContext(ctx, q, tid, sessionKey)
|
||||
rows, err := s.db.QueryContext(ctx, q, tid, rootAgentID, sessionKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -188,14 +221,38 @@ func (s *SQLiteSubagentTaskStore) ListBySession(
|
||||
return collectTasks(rows)
|
||||
}
|
||||
|
||||
// Archive marks old completed/failed/cancelled tasks as archived.
|
||||
func (s *SQLiteSubagentTaskStore) Archive(ctx context.Context, olderThan time.Duration) (int64, error) {
|
||||
// Archive marks a bounded batch of old terminal tasks as archived.
|
||||
func (s *SQLiteSubagentTaskStore) Archive(
|
||||
ctx context.Context, rootAgentID uuid.UUID, olderThan time.Duration, limit int,
|
||||
) (int64, error) {
|
||||
tid, err := requireTenantID(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if rootAgentID == uuid.Nil {
|
||||
return 0, store.ErrSubagentRootAgentIDRequired
|
||||
}
|
||||
if limit <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
cutoff := time.Now().UTC().Add(-olderThan).Format(time.RFC3339Nano)
|
||||
now := time.Now().UTC().Format(time.RFC3339Nano)
|
||||
q := `UPDATE subagent_tasks SET archived_at = ?, updated_at = ?
|
||||
WHERE status IN ('completed', 'failed', 'cancelled')
|
||||
AND archived_at IS NULL AND completed_at < ?`
|
||||
res, err := s.db.ExecContext(ctx, q, now, now, cutoff)
|
||||
WHERE tenant_id = ? AND root_agent_id = ? AND id IN (
|
||||
SELECT id
|
||||
FROM subagent_tasks
|
||||
WHERE tenant_id = ? AND root_agent_id = ?
|
||||
AND status IN ('completed', 'failed', 'cancelled')
|
||||
AND archived_at IS NULL AND completed_at < ?
|
||||
ORDER BY completed_at, id
|
||||
LIMIT ?
|
||||
)`
|
||||
res, err := s.db.ExecContext(
|
||||
ctx, q,
|
||||
now, now, tid, rootAgentID,
|
||||
tid, rootAgentID, cutoff, limit,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
@@ -204,11 +261,16 @@ func (s *SQLiteSubagentTaskStore) Archive(ctx context.Context, olderThan time.Du
|
||||
|
||||
// UpdateMetadata merges metadata keys atomically using json_set().
|
||||
// Builds a single UPDATE statement to avoid read-merge-write race window.
|
||||
func (s *SQLiteSubagentTaskStore) UpdateMetadata(ctx context.Context, id uuid.UUID, metadata map[string]any) error {
|
||||
func (s *SQLiteSubagentTaskStore) UpdateMetadata(
|
||||
ctx context.Context, rootAgentID, id uuid.UUID, metadata map[string]any,
|
||||
) error {
|
||||
tid, err := requireTenantID(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rootAgentID == uuid.Nil {
|
||||
return store.ErrSubagentRootAgentIDRequired
|
||||
}
|
||||
if len(metadata) == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -221,18 +283,46 @@ func (s *SQLiteSubagentTaskStore) UpdateMetadata(ctx context.Context, id uuid.UU
|
||||
if !validMetadataKey(k) {
|
||||
return fmt.Errorf("invalid metadata key: %q", k)
|
||||
}
|
||||
parts = append(parts, fmt.Sprintf("'$.%s', ?", k))
|
||||
parts = append(parts, fmt.Sprintf("'$.%s', json(?)", k))
|
||||
b, _ := json.Marshal(v)
|
||||
args = append(args, string(b))
|
||||
}
|
||||
|
||||
now := time.Now().UTC().Format(time.RFC3339Nano)
|
||||
setExpr := "json_set(metadata, " + strings.Join(parts, ", ") + ")"
|
||||
args = append(args, now, id, tid)
|
||||
args = append(args, now, id, tid, rootAgentID)
|
||||
|
||||
q := fmt.Sprintf(`UPDATE subagent_tasks SET metadata = %s, updated_at = ? WHERE id = ? AND tenant_id = ?`, setExpr)
|
||||
_, err = s.db.ExecContext(ctx, q, args...)
|
||||
return err
|
||||
q := fmt.Sprintf(`UPDATE subagent_tasks SET metadata = %s, updated_at = ?
|
||||
WHERE id = ? AND tenant_id = ? AND root_agent_id = ?`, setExpr)
|
||||
res, err := s.db.ExecContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
affected, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected != 1 {
|
||||
return store.ErrSubagentTaskNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RecoverInterrupted marks child runs that cannot survive a process restart as
|
||||
// failed. This cross-tenant update runs before the gateway accepts new work.
|
||||
func (s *SQLiteSubagentTaskStore) RecoverInterrupted(ctx context.Context) (int64, error) {
|
||||
if !store.IsMasterScope(ctx) {
|
||||
return 0, fmt.Errorf("recover interrupted subagent tasks requires master scope")
|
||||
}
|
||||
now := time.Now().UTC().Format(time.RFC3339Nano)
|
||||
q := `UPDATE subagent_tasks SET
|
||||
status = 'failed', result = ?, completed_at = ?, updated_at = ?
|
||||
WHERE status NOT IN ('completed', 'failed', 'cancelled')`
|
||||
res, err := s.db.ExecContext(ctx, q, store.InterruptedSubagentTaskResult, now, now)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return res.RowsAffected()
|
||||
}
|
||||
|
||||
func collectTasks(rows *sql.Rows) ([]store.SubagentTaskData, error) {
|
||||
|
||||
@@ -0,0 +1,586 @@
|
||||
//go:build sqlite || sqliteonly
|
||||
|
||||
package sqlitestore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
)
|
||||
|
||||
func createSQLiteSubagentTask(
|
||||
t *testing.T,
|
||||
taskStore *SQLiteSubagentTaskStore,
|
||||
ctx context.Context,
|
||||
rootAgentID uuid.UUID,
|
||||
rootAgentKey, sessionKey, status string,
|
||||
) uuid.UUID {
|
||||
t.Helper()
|
||||
|
||||
id := uuid.Must(uuid.NewV7())
|
||||
task := &store.SubagentTaskData{
|
||||
RootAgentID: rootAgentID,
|
||||
ParentAgentKey: rootAgentKey,
|
||||
SessionKey: &sessionKey,
|
||||
Subject: "store test",
|
||||
Description: "verify scoped persistence",
|
||||
Status: status,
|
||||
Depth: 1,
|
||||
Metadata: map[string]any{},
|
||||
}
|
||||
task.ID = id
|
||||
if err := taskStore.Create(ctx, task); err != nil {
|
||||
t.Fatalf("Create(%s): %v", id, err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func TestSQLiteSubagentTaskStoreRequiresTenantAndRootScope(t *testing.T) {
|
||||
db := newHookTestDB(t)
|
||||
tenantA, rootAID := seedHookTenantAgent(t, db)
|
||||
tenantB, tenantBRootID := seedHookTenantAgent(t, db)
|
||||
ctxA := sqliteTenantCtx(tenantA)
|
||||
ctxB := sqliteTenantCtx(tenantB)
|
||||
taskStore := NewSQLiteSubagentTaskStore(db)
|
||||
|
||||
const (
|
||||
rootA = "root-a"
|
||||
rootB = "root-b"
|
||||
sessionID = "shared-session"
|
||||
)
|
||||
rootBID := uuid.Must(uuid.NewV7())
|
||||
if _, err := db.Exec(
|
||||
`INSERT INTO agents (id, tenant_id, agent_key, agent_type, status, provider, model, owner_id)
|
||||
VALUES (?,?,?,'predefined','active','test','test-model','owner')`,
|
||||
rootBID, tenantA, rootB,
|
||||
); err != nil {
|
||||
t.Fatalf("seed second root agent: %v", err)
|
||||
}
|
||||
taskA := createSQLiteSubagentTask(t, taskStore, ctxA, rootAID, rootA, sessionID, "queued")
|
||||
taskB := createSQLiteSubagentTask(t, taskStore, ctxA, rootBID, rootB, sessionID, "queued")
|
||||
delegationTask := createSQLiteSubagentTask(t, taskStore, ctxA, rootAID, rootA, sessionID, "queued")
|
||||
if err := taskStore.UpdateMetadata(ctxA, rootAID, delegationTask, map[string]any{
|
||||
"completion_kind": "delegate",
|
||||
}); err != nil {
|
||||
t.Fatalf("mark delegation completion: %v", err)
|
||||
}
|
||||
_ = createSQLiteSubagentTask(t, taskStore, ctxB, tenantBRootID, rootA, sessionID, "queued")
|
||||
crossTenantTask := &store.SubagentTaskData{
|
||||
BaseModel: store.BaseModel{ID: uuid.Must(uuid.NewV7())},
|
||||
RootAgentID: tenantBRootID,
|
||||
ParentAgentKey: rootA,
|
||||
Subject: "cross tenant",
|
||||
Description: "must be rejected",
|
||||
Status: "queued",
|
||||
Depth: 1,
|
||||
}
|
||||
if err := taskStore.Create(ctxA, crossTenantTask); err == nil {
|
||||
t.Fatal("Create accepted root agent from another tenant")
|
||||
}
|
||||
|
||||
got, err := taskStore.Get(ctxA, rootAID, taskA)
|
||||
if err != nil {
|
||||
t.Fatalf("Get owning scope: %v", err)
|
||||
}
|
||||
if got == nil || got.ID != taskA {
|
||||
t.Fatalf("Get owning scope = %#v, want task %s", got, taskA)
|
||||
}
|
||||
|
||||
got, err = taskStore.Get(ctxA, rootBID, taskA)
|
||||
if err != nil {
|
||||
t.Fatalf("Get cross-root: %v", err)
|
||||
}
|
||||
if got != nil {
|
||||
t.Fatalf("Get cross-root = %#v, want nil", got)
|
||||
}
|
||||
|
||||
got, err = taskStore.Get(ctxB, rootAID, taskA)
|
||||
if err != nil {
|
||||
t.Fatalf("Get cross-tenant: %v", err)
|
||||
}
|
||||
if got != nil {
|
||||
t.Fatalf("Get cross-tenant = %#v, want nil", got)
|
||||
}
|
||||
|
||||
if err := taskStore.UpdateStatus(ctxA, rootBID, taskA, "completed", nil, 3, 10, 20); !errors.Is(err, store.ErrSubagentTaskNotFound) {
|
||||
t.Fatalf("UpdateStatus cross-root error = %v, want scoped not found", err)
|
||||
}
|
||||
got, err = taskStore.Get(ctxA, rootAID, taskA)
|
||||
if err != nil {
|
||||
t.Fatalf("Get after cross-root status update: %v", err)
|
||||
}
|
||||
if got.Status != "queued" || got.CompletedAt != nil {
|
||||
t.Fatalf("cross-root status update changed task: status=%q completed_at=%v", got.Status, got.CompletedAt)
|
||||
}
|
||||
|
||||
if err := taskStore.UpdateMetadata(ctxA, rootBID, taskA, map[string]any{"denied": true}); !errors.Is(err, store.ErrSubagentTaskNotFound) {
|
||||
t.Fatalf("UpdateMetadata cross-root error = %v, want scoped not found", err)
|
||||
}
|
||||
got, err = taskStore.Get(ctxA, rootAID, taskA)
|
||||
if err != nil {
|
||||
t.Fatalf("Get after cross-root metadata update: %v", err)
|
||||
}
|
||||
if _, exists := got.Metadata["denied"]; exists {
|
||||
t.Fatalf("cross-root metadata update changed task: %#v", got.Metadata)
|
||||
}
|
||||
if err := taskStore.UpdateMetadata(ctxA, rootAID, taskA, map[string]any{
|
||||
"announcement_status": "undelivered",
|
||||
"delivered": false,
|
||||
}); err != nil {
|
||||
t.Fatalf("UpdateMetadata owning scope: %v", err)
|
||||
}
|
||||
got, err = taskStore.Get(ctxA, rootAID, taskA)
|
||||
if err != nil {
|
||||
t.Fatalf("Get after owning metadata update: %v", err)
|
||||
}
|
||||
if got.Metadata["announcement_status"] != "undelivered" || got.Metadata["delivered"] != false {
|
||||
t.Fatalf("metadata JSON types were not preserved: %#v", got.Metadata)
|
||||
}
|
||||
|
||||
parentTasksA, err := taskStore.ListByParent(ctxA, rootAID, "")
|
||||
if err != nil {
|
||||
t.Fatalf("ListByParent root A: %v", err)
|
||||
}
|
||||
if len(parentTasksA) != 1 || parentTasksA[0].ID != taskA {
|
||||
t.Fatalf("ListByParent root A = %#v, want only self-clone %s", parentTasksA, taskA)
|
||||
}
|
||||
parentTasksB, err := taskStore.ListByParent(ctxA, rootBID, "queued")
|
||||
if err != nil {
|
||||
t.Fatalf("ListByParent root B: %v", err)
|
||||
}
|
||||
if len(parentTasksB) != 1 || parentTasksB[0].ID != taskB {
|
||||
t.Fatalf("ListByParent root B = %#v, want only %s", parentTasksB, taskB)
|
||||
}
|
||||
|
||||
tasksA, err := taskStore.ListBySession(ctxA, rootAID, sessionID)
|
||||
if err != nil {
|
||||
t.Fatalf("ListBySession root A: %v", err)
|
||||
}
|
||||
if len(tasksA) != 1 || tasksA[0].ID != taskA {
|
||||
t.Fatalf("ListBySession root A = %#v, want only self-clone %s", tasksA, taskA)
|
||||
}
|
||||
tasksB, err := taskStore.ListBySession(ctxA, rootBID, sessionID)
|
||||
if err != nil {
|
||||
t.Fatalf("ListBySession root B: %v", err)
|
||||
}
|
||||
if len(tasksB) != 1 || tasksB[0].ID != taskB {
|
||||
t.Fatalf("ListBySession root B = %#v, want only %s", tasksB, taskB)
|
||||
}
|
||||
|
||||
if _, err := taskStore.Get(context.Background(), rootAID, taskA); err == nil {
|
||||
t.Fatal("Get without tenant context returned nil error")
|
||||
}
|
||||
if _, err := taskStore.Get(ctxA, uuid.Nil, taskA); !errors.Is(err, store.ErrSubagentRootAgentIDRequired) {
|
||||
t.Fatalf("Get empty root error = %v, want %v", err, store.ErrSubagentRootAgentIDRequired)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteSubagentTaskStoreRejectsRecreatedAgentWithSameKey(t *testing.T) {
|
||||
db := newHookTestDB(t)
|
||||
tenantID, oldRootAgentID := seedHookTenantAgent(t, db)
|
||||
ctx := sqliteTenantCtx(tenantID)
|
||||
taskStore := NewSQLiteSubagentTaskStore(db)
|
||||
|
||||
const rootAgentKey = "recreated-root"
|
||||
if _, err := db.Exec(
|
||||
`UPDATE agents SET agent_key = ? WHERE id = ?`,
|
||||
rootAgentKey, oldRootAgentID,
|
||||
); err != nil {
|
||||
t.Fatalf("rename original root agent: %v", err)
|
||||
}
|
||||
taskID := createSQLiteSubagentTask(
|
||||
t, taskStore, ctx, oldRootAgentID, rootAgentKey, "recreated-root-session", "queued",
|
||||
)
|
||||
|
||||
if _, err := db.Exec(`DELETE FROM agents WHERE id = ?`, oldRootAgentID); err != nil {
|
||||
t.Fatalf("delete original root agent: %v", err)
|
||||
}
|
||||
newRootAgentID := uuid.Must(uuid.NewV7())
|
||||
if _, err := db.Exec(
|
||||
`INSERT INTO agents (id, tenant_id, agent_key, agent_type, status, provider, model, owner_id)
|
||||
VALUES (?,?,?,'predefined','active','test','test-model','owner')`,
|
||||
newRootAgentID, tenantID, rootAgentKey,
|
||||
); err != nil {
|
||||
t.Fatalf("recreate root agent: %v", err)
|
||||
}
|
||||
|
||||
got, err := taskStore.Get(ctx, newRootAgentID, taskID)
|
||||
if err != nil {
|
||||
t.Fatalf("Get with recreated root agent: %v", err)
|
||||
}
|
||||
if got != nil {
|
||||
t.Fatalf("Get with recreated root agent returned old task %s", got.ID)
|
||||
}
|
||||
if err := taskStore.UpdateStatus(ctx, newRootAgentID, taskID, "completed", nil, 1, 2, 3); !errors.Is(err, store.ErrSubagentTaskNotFound) {
|
||||
t.Fatalf("UpdateStatus with recreated root agent error = %v, want scoped not found", err)
|
||||
}
|
||||
var status string
|
||||
var rootAgentID sql.NullString
|
||||
if err := db.QueryRow(
|
||||
`SELECT status, root_agent_id FROM subagent_tasks WHERE id = ?`, taskID,
|
||||
).Scan(&status, &rootAgentID); err != nil {
|
||||
t.Fatalf("read preserved old task: %v", err)
|
||||
}
|
||||
if status != "queued" || rootAgentID.Valid {
|
||||
t.Fatalf("old task after recreate: status=%q root_agent_id=%q, want queued/NULL", status, rootAgentID.String)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteSubagentTaskMigrationBackfillsOnlySafeOwners(t *testing.T) {
|
||||
db := newHookTestDB(t)
|
||||
tenantID, oldRootAgentID := seedHookTenantAgent(t, db)
|
||||
|
||||
const rootAgentKey = "migration-recreated-root"
|
||||
if _, err := db.Exec(
|
||||
`UPDATE agents SET agent_key = ?, deleted_at = ? WHERE id = ?`,
|
||||
rootAgentKey, time.Now().UTC().Format(time.RFC3339Nano), oldRootAgentID,
|
||||
); err != nil {
|
||||
t.Fatalf("prepare historical root agent: %v", err)
|
||||
}
|
||||
newRootAgentID := uuid.Must(uuid.NewV7())
|
||||
if _, err := db.Exec(
|
||||
`INSERT INTO agents (id, tenant_id, agent_key, agent_type, status, provider, model, owner_id)
|
||||
VALUES (?,?,?,'predefined','active','test','test-model','owner')`,
|
||||
newRootAgentID, tenantID, rootAgentKey,
|
||||
); err != nil {
|
||||
t.Fatalf("insert recreated root agent: %v", err)
|
||||
}
|
||||
|
||||
const (
|
||||
safeRootKey = "migration-safe-root"
|
||||
recreatedOnlyKey = "migration-hard-deleted-root"
|
||||
)
|
||||
safeRootAgentID := uuid.Must(uuid.NewV7())
|
||||
recreatedOnlyAgentID := uuid.Must(uuid.NewV7())
|
||||
if _, err := db.Exec(
|
||||
`INSERT INTO agents
|
||||
(id, tenant_id, agent_key, agent_type, status, provider, model, owner_id, created_at)
|
||||
VALUES (?,?,?,'predefined','active','test','test-model','owner',?),
|
||||
(?,?,?,'predefined','active','test','test-model','owner',?)`,
|
||||
safeRootAgentID, tenantID, safeRootKey, "2026-01-01T00:00:00Z",
|
||||
recreatedOnlyAgentID, tenantID, recreatedOnlyKey, "2026-03-01T00:00:00Z",
|
||||
); err != nil {
|
||||
t.Fatalf("insert temporal migration agents: %v", err)
|
||||
}
|
||||
|
||||
metadataTaskID := uuid.Must(uuid.NewV7())
|
||||
ambiguousTaskID := uuid.Must(uuid.NewV7())
|
||||
safeTaskID := uuid.Must(uuid.NewV7())
|
||||
recreatedTaskID := uuid.Must(uuid.NewV7())
|
||||
if _, err := db.Exec(
|
||||
`INSERT INTO subagent_tasks
|
||||
(id, tenant_id, parent_agent_key, subject, description, status, metadata, created_at)
|
||||
VALUES (?,?,?,?,?,'queued',?,?),
|
||||
(?,?,?,?,?,'queued','{}',?),
|
||||
(?,?,?,?,?,'queued','{}',?),
|
||||
(?,?,?,?,?,'queued','{}',?)`,
|
||||
metadataTaskID, tenantID, rootAgentKey, "metadata", "metadata owner",
|
||||
fmt.Sprintf(`{"root_agent_id":%q}`, oldRootAgentID.String()), "2026-04-01T00:00:00Z",
|
||||
ambiguousTaskID, tenantID, rootAgentKey, "ambiguous", "ambiguous owner", "2026-04-01T00:00:00Z",
|
||||
safeTaskID, tenantID, safeRootKey, "safe", "safe temporal owner", "2026-02-01T00:00:00Z",
|
||||
recreatedTaskID, tenantID, recreatedOnlyKey, "recreated", "older than current owner", "2026-02-01T00:00:00Z",
|
||||
); err != nil {
|
||||
t.Fatalf("insert legacy tasks: %v", err)
|
||||
}
|
||||
|
||||
if _, err := db.Exec(`
|
||||
DROP TRIGGER trg_subagent_tasks_root_tenant_insert;
|
||||
DROP TRIGGER trg_subagent_tasks_root_tenant_update;
|
||||
DROP INDEX idx_subagent_tasks_root_archive;
|
||||
DROP INDEX idx_subagent_tasks_root_session;
|
||||
DROP INDEX idx_subagent_tasks_root_status;
|
||||
ALTER TABLE subagent_tasks DROP COLUMN root_agent_id;
|
||||
UPDATE schema_version SET version = 58;
|
||||
`); err != nil {
|
||||
t.Fatalf("prepare v58 schema: %v", err)
|
||||
}
|
||||
if err := EnsureSchema(db); err != nil {
|
||||
t.Fatalf("migrate v58 to v59: %v", err)
|
||||
}
|
||||
|
||||
var metadataOwner sql.NullString
|
||||
if err := db.QueryRow(
|
||||
`SELECT root_agent_id FROM subagent_tasks WHERE id = ?`, metadataTaskID,
|
||||
).Scan(&metadataOwner); err != nil {
|
||||
t.Fatalf("read metadata-owned task: %v", err)
|
||||
}
|
||||
if !metadataOwner.Valid || metadataOwner.String != oldRootAgentID.String() {
|
||||
t.Fatalf("metadata owner = %q, want %s", metadataOwner.String, oldRootAgentID)
|
||||
}
|
||||
var ambiguousOwner sql.NullString
|
||||
if err := db.QueryRow(
|
||||
`SELECT root_agent_id FROM subagent_tasks WHERE id = ?`, ambiguousTaskID,
|
||||
).Scan(&ambiguousOwner); err != nil {
|
||||
t.Fatalf("read ambiguous task: %v", err)
|
||||
}
|
||||
if ambiguousOwner.Valid {
|
||||
t.Fatalf("ambiguous key owner = %q, want NULL", ambiguousOwner.String)
|
||||
}
|
||||
var safeOwner sql.NullString
|
||||
if err := db.QueryRow(
|
||||
`SELECT root_agent_id FROM subagent_tasks WHERE id = ?`, safeTaskID,
|
||||
).Scan(&safeOwner); err != nil {
|
||||
t.Fatalf("read safely backfilled task: %v", err)
|
||||
}
|
||||
if !safeOwner.Valid || safeOwner.String != safeRootAgentID.String() {
|
||||
t.Fatalf("safe owner = %q, want %s", safeOwner.String, safeRootAgentID)
|
||||
}
|
||||
var recreatedOwner sql.NullString
|
||||
if err := db.QueryRow(
|
||||
`SELECT root_agent_id FROM subagent_tasks WHERE id = ?`, recreatedTaskID,
|
||||
).Scan(&recreatedOwner); err != nil {
|
||||
t.Fatalf("read recreated-owner task: %v", err)
|
||||
}
|
||||
if recreatedOwner.Valid {
|
||||
t.Fatalf("new same-key agent inherited older task: owner=%q", recreatedOwner.String)
|
||||
}
|
||||
var version int
|
||||
if err := db.QueryRow(`SELECT version FROM schema_version`).Scan(&version); err != nil {
|
||||
t.Fatalf("read migrated schema version: %v", err)
|
||||
}
|
||||
if version != SchemaVersion {
|
||||
t.Fatalf("schema version = %d, want %d", version, SchemaVersion)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteSubagentTaskStoreCompletedAtOnlyForTerminalStatus(t *testing.T) {
|
||||
db := newHookTestDB(t)
|
||||
tenantID, rootAgentID := seedHookTenantAgent(t, db)
|
||||
ctx := sqliteTenantCtx(tenantID)
|
||||
taskStore := NewSQLiteSubagentTaskStore(db)
|
||||
|
||||
tests := []struct {
|
||||
status string
|
||||
terminal bool
|
||||
}{
|
||||
{status: "new"},
|
||||
{status: "queued"},
|
||||
{status: "running"},
|
||||
{status: "waiting_child"},
|
||||
{status: "completed", terminal: true},
|
||||
{status: "failed", terminal: true},
|
||||
{status: "cancelled", terminal: true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.status, func(t *testing.T) {
|
||||
rootAgentKey := "root-" + tt.status
|
||||
id := createSQLiteSubagentTask(t, taskStore, ctx, rootAgentID, rootAgentKey, "session-"+tt.status, "queued")
|
||||
if err := taskStore.UpdateStatus(ctx, rootAgentID, id, tt.status, nil, 0, 0, 0); err != nil {
|
||||
t.Fatalf("UpdateStatus(%q): %v", tt.status, err)
|
||||
}
|
||||
got, err := taskStore.Get(ctx, rootAgentID, id)
|
||||
if err != nil {
|
||||
t.Fatalf("Get(%q): %v", tt.status, err)
|
||||
}
|
||||
if (got.CompletedAt != nil) != tt.terminal {
|
||||
t.Fatalf("status %q completed_at = %v, terminal=%v", tt.status, got.CompletedAt, tt.terminal)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteSubagentTaskStoreRecoverInterrupted(t *testing.T) {
|
||||
db := newHookTestDB(t)
|
||||
tenantA, rootAID := seedHookTenantAgent(t, db)
|
||||
tenantB, rootBID := seedHookTenantAgent(t, db)
|
||||
ctxA := sqliteTenantCtx(tenantA)
|
||||
ctxB := sqliteTenantCtx(tenantB)
|
||||
taskStore := NewSQLiteSubagentTaskStore(db)
|
||||
|
||||
queuedID := createSQLiteSubagentTask(t, taskStore, ctxA, rootAID, "root-a", "queued", "queued")
|
||||
if _, err := db.Exec(
|
||||
`UPDATE subagent_tasks SET completed_at = ? WHERE id = ?`,
|
||||
time.Now().UTC().Add(-time.Hour).Format(time.RFC3339Nano),
|
||||
queuedID,
|
||||
); err != nil {
|
||||
t.Fatalf("seed malformed queued completed_at: %v", err)
|
||||
}
|
||||
runningID := createSQLiteSubagentTask(t, taskStore, ctxA, rootAID, "root-a", "running", "running")
|
||||
waitingID := createSQLiteSubagentTask(
|
||||
t, taskStore, ctxB, rootBID, "root-b", "waiting", "waiting_child",
|
||||
)
|
||||
if err := taskStore.UpdateMetadata(ctxB, rootBID, waitingID, map[string]any{
|
||||
"completion_kind": "delegate",
|
||||
"completion_media": []map[string]any{{
|
||||
"path": ".delegations/completed-before-crash/report.pdf",
|
||||
"mime_type": "application/pdf",
|
||||
}},
|
||||
}); err != nil {
|
||||
t.Fatalf("record published artifact metadata: %v", err)
|
||||
}
|
||||
completedID := createSQLiteSubagentTask(t, taskStore, ctxB, rootBID, "root-b", "completed", "queued")
|
||||
completedResult := "already completed"
|
||||
if err := taskStore.UpdateStatus(
|
||||
ctxB, rootBID, completedID, "completed", &completedResult, 1, 2, 3,
|
||||
); err != nil {
|
||||
t.Fatalf("complete terminal task: %v", err)
|
||||
}
|
||||
|
||||
if _, err := taskStore.RecoverInterrupted(ctxA); err == nil {
|
||||
t.Fatal("RecoverInterrupted accepted tenant-scoped context")
|
||||
}
|
||||
recoveryCtx := store.WithTenantID(context.Background(), store.MasterTenantID)
|
||||
recovered, err := taskStore.RecoverInterrupted(recoveryCtx)
|
||||
if err != nil {
|
||||
t.Fatalf("RecoverInterrupted: %v", err)
|
||||
}
|
||||
if recovered != 3 {
|
||||
t.Fatalf("RecoverInterrupted recovered %d tasks, want 3", recovered)
|
||||
}
|
||||
|
||||
for _, item := range []struct {
|
||||
ctx context.Context
|
||||
root uuid.UUID
|
||||
id uuid.UUID
|
||||
}{
|
||||
{ctx: ctxA, root: rootAID, id: queuedID},
|
||||
{ctx: ctxA, root: rootAID, id: runningID},
|
||||
{ctx: ctxB, root: rootBID, id: waitingID},
|
||||
} {
|
||||
got, getErr := taskStore.Get(item.ctx, item.root, item.id)
|
||||
if getErr != nil {
|
||||
t.Fatalf("Get recovered task %s: %v", item.id, getErr)
|
||||
}
|
||||
if got.Status != "failed" || got.CompletedAt == nil || got.Result == nil ||
|
||||
!strings.Contains(*got.Result, "gateway stopped") {
|
||||
t.Fatalf("recovered task %s = %#v", item.id, got)
|
||||
}
|
||||
if item.id == waitingID {
|
||||
if got.Metadata["completion_kind"] != "delegate" ||
|
||||
got.Metadata["completion_media"] == nil {
|
||||
t.Fatalf("published artifact metadata was lost: %#v", got.Metadata)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
completed, err := taskStore.Get(ctxB, rootBID, completedID)
|
||||
if err != nil {
|
||||
t.Fatalf("Get completed task: %v", err)
|
||||
}
|
||||
if completed.Status != "completed" || completed.Result == nil ||
|
||||
*completed.Result != completedResult {
|
||||
t.Fatalf("completed task changed during recovery: %#v", completed)
|
||||
}
|
||||
|
||||
recovered, err = taskStore.RecoverInterrupted(recoveryCtx)
|
||||
if err != nil {
|
||||
t.Fatalf("RecoverInterrupted second pass: %v", err)
|
||||
}
|
||||
if recovered != 0 {
|
||||
t.Fatalf("RecoverInterrupted second pass recovered %d tasks, want 0", recovered)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteSubagentTaskStoreArchiveIsScopedAndBounded(t *testing.T) {
|
||||
db := newHookTestDB(t)
|
||||
tenantA, rootAID := seedHookTenantAgent(t, db)
|
||||
tenantB, tenantBRootID := seedHookTenantAgent(t, db)
|
||||
ctxA := sqliteTenantCtx(tenantA)
|
||||
ctxB := sqliteTenantCtx(tenantB)
|
||||
taskStore := NewSQLiteSubagentTaskStore(db)
|
||||
|
||||
const (
|
||||
rootA = "root-a"
|
||||
rootB = "root-b"
|
||||
)
|
||||
rootBID := uuid.Must(uuid.NewV7())
|
||||
if _, err := db.Exec(
|
||||
`INSERT INTO agents (id, tenant_id, agent_key, agent_type, status, provider, model, owner_id)
|
||||
VALUES (?,?,?,'predefined','active','test','test-model','owner')`,
|
||||
rootBID, tenantA, rootB,
|
||||
); err != nil {
|
||||
t.Fatalf("seed second root agent: %v", err)
|
||||
}
|
||||
var rootATasks []uuid.UUID
|
||||
for i := 0; i < 3; i++ {
|
||||
rootATasks = append(rootATasks, createSQLiteSubagentTask(
|
||||
t, taskStore, ctxA, rootAID, rootA, fmt.Sprintf("session-a-%d", i), "queued",
|
||||
))
|
||||
}
|
||||
rootBTask := createSQLiteSubagentTask(t, taskStore, ctxA, rootBID, rootB, "session-b", "queued")
|
||||
tenantBTask := createSQLiteSubagentTask(t, taskStore, ctxB, tenantBRootID, rootA, "session-other-tenant", "queued")
|
||||
queuedTask := createSQLiteSubagentTask(t, taskStore, ctxA, rootAID, rootA, "session-queued", "queued")
|
||||
|
||||
for _, item := range []struct {
|
||||
ctx context.Context
|
||||
root uuid.UUID
|
||||
id uuid.UUID
|
||||
}{
|
||||
{ctx: ctxA, root: rootAID, id: rootATasks[0]},
|
||||
{ctx: ctxA, root: rootAID, id: rootATasks[1]},
|
||||
{ctx: ctxA, root: rootAID, id: rootATasks[2]},
|
||||
{ctx: ctxA, root: rootBID, id: rootBTask},
|
||||
{ctx: ctxB, root: tenantBRootID, id: tenantBTask},
|
||||
} {
|
||||
if err := taskStore.UpdateStatus(item.ctx, item.root, item.id, "completed", nil, 0, 0, 0); err != nil {
|
||||
t.Fatalf("UpdateStatus(%s): %v", item.id, err)
|
||||
}
|
||||
}
|
||||
|
||||
oldTime := time.Now().UTC().Add(-2 * time.Hour).Format(time.RFC3339Nano)
|
||||
for _, id := range append(append([]uuid.UUID{}, rootATasks...), rootBTask, tenantBTask) {
|
||||
if _, err := db.Exec(`UPDATE subagent_tasks SET completed_at = ? WHERE id = ?`, oldTime, id); err != nil {
|
||||
t.Fatalf("backdate terminal task %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
// A non-terminal row must remain unarchived even if malformed legacy data
|
||||
// happens to carry a completed_at value.
|
||||
if _, err := db.Exec(`UPDATE subagent_tasks SET completed_at = ? WHERE id = ?`, oldTime, queuedTask); err != nil {
|
||||
t.Fatalf("backdate queued task: %v", err)
|
||||
}
|
||||
|
||||
archived, err := taskStore.Archive(ctxA, rootAID, time.Hour, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("Archive first batch: %v", err)
|
||||
}
|
||||
if archived != 2 {
|
||||
t.Fatalf("Archive first batch affected %d rows, want 2", archived)
|
||||
}
|
||||
|
||||
assertSQLiteArchivedCount(t, db, tenantA, rootAID, 2)
|
||||
assertSQLiteArchivedCount(t, db, tenantA, rootBID, 0)
|
||||
assertSQLiteArchivedCount(t, db, tenantB, tenantBRootID, 0)
|
||||
|
||||
archived, err = taskStore.Archive(ctxA, rootAID, time.Hour, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("Archive second batch: %v", err)
|
||||
}
|
||||
if archived != 1 {
|
||||
t.Fatalf("Archive second batch affected %d rows, want 1", archived)
|
||||
}
|
||||
assertSQLiteArchivedCount(t, db, tenantA, rootAID, 3)
|
||||
|
||||
var queuedArchived sql.NullString
|
||||
if err := db.QueryRow(`SELECT archived_at FROM subagent_tasks WHERE id = ?`, queuedTask).Scan(&queuedArchived); err != nil {
|
||||
t.Fatalf("read queued archived_at: %v", err)
|
||||
}
|
||||
if queuedArchived.Valid {
|
||||
t.Fatalf("queued task archived_at = %q, want NULL", queuedArchived.String)
|
||||
}
|
||||
}
|
||||
|
||||
func assertSQLiteArchivedCount(
|
||||
t *testing.T, db *sql.DB, tenantID, rootAgentID uuid.UUID, want int,
|
||||
) {
|
||||
t.Helper()
|
||||
var got int
|
||||
if err := db.QueryRow(
|
||||
`SELECT COUNT(*) FROM subagent_tasks
|
||||
WHERE tenant_id = ? AND root_agent_id = ? AND archived_at IS NOT NULL`,
|
||||
tenantID, rootAgentID,
|
||||
).Scan(&got); err != nil {
|
||||
t.Fatalf("count archived tasks for %s/%s: %v", tenantID, rootAgentID, err)
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("archived tasks for %s/%s = %d, want %d", tenantID, rootAgentID, got, want)
|
||||
}
|
||||
}
|
||||
@@ -40,6 +40,7 @@ type Stores struct {
|
||||
SkillEvolution SkillEvolutionStore
|
||||
SystemConfigs SystemConfigStore
|
||||
SubagentTasks SubagentTaskStore
|
||||
SubagentTaskRecovery SubagentTaskRecoveryStore
|
||||
Vault VaultStore
|
||||
Episodic EpisodicStore
|
||||
EvolutionMetrics EvolutionMetricsStore
|
||||
|
||||
@@ -2,15 +2,39 @@ package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrSubagentRootAgentIDRequired = errors.New("subagent root agent ID required")
|
||||
ErrSubagentTaskNotFound = errors.New("subagent task not found in owner scope")
|
||||
)
|
||||
|
||||
const (
|
||||
InterruptedSubagentTaskResult = "interrupted: gateway stopped before terminal completion was durably recorded"
|
||||
// SubagentTaskStatusMaxLength matches the durable PG and SQLite schema contract.
|
||||
SubagentTaskStatusMaxLength = 20
|
||||
)
|
||||
|
||||
// IsTerminalSubagentTaskStatus reports whether a status ends the task
|
||||
// lifecycle and therefore receives a completed_at timestamp.
|
||||
func IsTerminalSubagentTaskStatus(status string) bool {
|
||||
switch status {
|
||||
case "completed", "failed", "cancelled":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// SubagentTaskData represents a persisted subagent task for audit trail and cost attribution.
|
||||
type SubagentTaskData struct {
|
||||
BaseModel
|
||||
TenantID uuid.UUID `json:"tenant_id" db:"tenant_id"`
|
||||
RootAgentID uuid.UUID `json:"root_agent_id" db:"root_agent_id"`
|
||||
ParentAgentKey string `json:"parent_agent_key" db:"parent_agent_key"`
|
||||
SessionKey *string `json:"session_key,omitempty" db:"session_key"`
|
||||
Subject string `json:"subject" db:"subject"`
|
||||
@@ -34,29 +58,39 @@ type SubagentTaskData struct {
|
||||
}
|
||||
|
||||
// SubagentTaskStore persists subagent task lifecycle for audit trail and cost attribution.
|
||||
// In-memory SubagentManager remains the source of truth for active operations;
|
||||
// DB writes are fire-and-forget (non-blocking).
|
||||
// In-memory SubagentManager remains the source of truth for active operations.
|
||||
type SubagentTaskStore interface {
|
||||
// Create persists a new subagent task at spawn time.
|
||||
Create(ctx context.Context, task *SubagentTaskData) error
|
||||
|
||||
// Get retrieves a single task by ID (tenant-scoped).
|
||||
Get(ctx context.Context, id uuid.UUID) (*SubagentTaskData, error)
|
||||
// Get retrieves a task owned by the tenant and immutable root-agent UUID.
|
||||
Get(ctx context.Context, rootAgentID, id uuid.UUID) (*SubagentTaskData, error)
|
||||
|
||||
// UpdateStatus updates status, result, iterations, and token counts on completion/failure.
|
||||
UpdateStatus(ctx context.Context, id uuid.UUID, status string, result *string, iterations int, inputTokens, outputTokens int64) error
|
||||
UpdateStatus(ctx context.Context, rootAgentID, id uuid.UUID, status string, result *string, iterations int, inputTokens, outputTokens int64) error
|
||||
|
||||
// ListByParent returns tasks for a parent agent key, optionally filtered by status.
|
||||
// ListByParent returns tasks owned by a root-agent UUID, optionally filtered by status.
|
||||
// Empty statusFilter returns all statuses. Ordered by created_at DESC.
|
||||
ListByParent(ctx context.Context, parentAgentKey string, statusFilter string) ([]SubagentTaskData, error)
|
||||
ListByParent(ctx context.Context, rootAgentID uuid.UUID, statusFilter string) ([]SubagentTaskData, error)
|
||||
|
||||
// ListBySession returns tasks for a specific session key (tenant-scoped).
|
||||
ListBySession(ctx context.Context, sessionKey string) ([]SubagentTaskData, error)
|
||||
// ListBySession returns tasks for a session owned by the tenant and immutable root-agent UUID.
|
||||
ListBySession(ctx context.Context, rootAgentID uuid.UUID, sessionKey string) ([]SubagentTaskData, error)
|
||||
|
||||
// Archive marks old completed/failed/cancelled tasks as archived.
|
||||
// Returns the number of rows affected.
|
||||
Archive(ctx context.Context, olderThan time.Duration) (int64, error)
|
||||
// Archive marks at most limit old terminal tasks owned by the tenant and
|
||||
// immutable root-agent UUID as archived. Returns the number of rows affected.
|
||||
Archive(ctx context.Context, rootAgentID uuid.UUID, olderThan time.Duration, limit int) (int64, error)
|
||||
|
||||
// UpdateMetadata merges metadata on an existing task.
|
||||
UpdateMetadata(ctx context.Context, id uuid.UUID, metadata map[string]any) error
|
||||
UpdateMetadata(ctx context.Context, rootAgentID, id uuid.UUID, metadata map[string]any) error
|
||||
}
|
||||
|
||||
// SubagentTaskRecoveryStore exposes the startup-only, cross-tenant maintenance
|
||||
// operation separately from tenant/root-scoped task CRUD.
|
||||
type SubagentTaskRecoveryStore interface {
|
||||
// RecoverInterrupted marks every non-terminal task from the previous
|
||||
// process as failed. It is a cross-tenant startup operation and must run
|
||||
// before the gateway accepts new child runs. The current gateway
|
||||
// architecture is single-process; multi-replica deployments would need
|
||||
// coordinated ownership before invoking this recovery.
|
||||
RecoverInterrupted(ctx context.Context) (int64, error)
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -11,17 +13,23 @@ import (
|
||||
"github.com/nextlevelbuilder/goclaw/internal/bus"
|
||||
)
|
||||
|
||||
var ErrAnnounceQueueDrainTimeout = errors.New("announce_queue_drain_timeout")
|
||||
|
||||
// AnnounceQueueItem represents a single subagent result waiting to be announced.
|
||||
type AnnounceQueueItem struct {
|
||||
SubagentID string
|
||||
Label string
|
||||
Status string // "completed", "failed", "cancelled"
|
||||
Result string
|
||||
Media []bus.MediaFile // media files from tool results
|
||||
Runtime time.Duration
|
||||
Iterations int
|
||||
InputTokens int64
|
||||
OutputTokens int64
|
||||
SubagentID string
|
||||
CompletionID uuid.UUID
|
||||
DurablyPersisted bool
|
||||
ParentTaskID string
|
||||
Depth int
|
||||
Label string
|
||||
Status string // "completed", "failed", "cancelled"
|
||||
Result string
|
||||
Media []bus.MediaFile // media files from tool results
|
||||
Runtime time.Duration
|
||||
Iterations int
|
||||
InputTokens int64
|
||||
OutputTokens int64
|
||||
}
|
||||
|
||||
// AnnounceMetadata carries origin info for routing the batched announce.
|
||||
@@ -31,10 +39,11 @@ type AnnounceMetadata struct {
|
||||
OriginPeerKind string
|
||||
OriginLocalKey string // composite key with topic/thread suffix for routing
|
||||
OriginUserID string
|
||||
OriginSenderID string // real acting sender; preserves permission attribution through re-ingress (#915)
|
||||
OriginRole string // caller's RBAC role; bypasses per-user grants for admin/operator/owner (#915)
|
||||
OriginSessionKey string // exact parent session key (WS uses non-standard format)
|
||||
OriginSenderID string // real acting sender; preserves permission attribution through re-ingress (#915)
|
||||
OriginRole string // caller's RBAC role; bypasses per-user grants for admin/operator/owner (#915)
|
||||
OriginSessionKey string // exact parent session key (WS uses non-standard format)
|
||||
OriginTenantID uuid.UUID // parent tenant for announce routing
|
||||
RootAgentID uuid.UUID
|
||||
ParentAgent string
|
||||
OriginTraceID string // parent trace UUID for announce linking
|
||||
OriginRootSpanID string // parent agent's root span UUID
|
||||
@@ -47,7 +56,9 @@ type AnnounceQueue struct {
|
||||
queues map[string]*sessionQueue // session key → queue
|
||||
debounce time.Duration // default 1000ms
|
||||
cap int // max items per session before immediate drain (default 20)
|
||||
onDrain func(sessionKey string, items []AnnounceQueueItem, meta AnnounceMetadata)
|
||||
onDrain func(sessionKey string, items []AnnounceQueueItem, meta AnnounceMetadata)
|
||||
closed bool
|
||||
drainWG sync.WaitGroup
|
||||
}
|
||||
|
||||
type sessionQueue struct {
|
||||
@@ -81,6 +92,9 @@ func NewAnnounceQueue(
|
||||
func (aq *AnnounceQueue) Enqueue(sessionKey string, item AnnounceQueueItem, meta AnnounceMetadata) {
|
||||
aq.mu.Lock()
|
||||
defer aq.mu.Unlock()
|
||||
if aq.closed {
|
||||
return
|
||||
}
|
||||
|
||||
sq, ok := aq.queues[sessionKey]
|
||||
if !ok {
|
||||
@@ -98,7 +112,7 @@ func (aq *AnnounceQueue) Enqueue(sessionKey string, item AnnounceQueueItem, meta
|
||||
items := sq.items
|
||||
sqMeta := sq.meta
|
||||
delete(aq.queues, sessionKey)
|
||||
go aq.drain(sessionKey, items, sqMeta)
|
||||
aq.startDrainLocked(sessionKey, items, sqMeta)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -116,12 +130,62 @@ func (aq *AnnounceQueue) Enqueue(sessionKey string, item AnnounceQueueItem, meta
|
||||
items := sq.items
|
||||
sqMeta := sq.meta
|
||||
delete(aq.queues, sessionKey)
|
||||
aq.startDrainLocked(sessionKey, items, sqMeta)
|
||||
aq.mu.Unlock()
|
||||
|
||||
aq.drain(sessionKey, items, sqMeta)
|
||||
})
|
||||
}
|
||||
|
||||
func (aq *AnnounceQueue) startDrainLocked(
|
||||
sessionKey string,
|
||||
items []AnnounceQueueItem,
|
||||
meta AnnounceMetadata,
|
||||
) {
|
||||
if aq.closed {
|
||||
return
|
||||
}
|
||||
aq.drainWG.Add(1)
|
||||
go func() {
|
||||
defer aq.drainWG.Done()
|
||||
aq.drain(sessionKey, items, meta)
|
||||
}()
|
||||
}
|
||||
|
||||
// CloseContext closes intake, drops pending debounce batches, and waits for any
|
||||
// drain callback that already started. Dropping pending parent-resume messages
|
||||
// during process shutdown avoids starting new agent runs against dependencies
|
||||
// that are being torn down; task terminal state remains durable.
|
||||
func (aq *AnnounceQueue) CloseContext(ctx context.Context) error {
|
||||
if aq == nil {
|
||||
return nil
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
aq.mu.Lock()
|
||||
if !aq.closed {
|
||||
aq.closed = true
|
||||
for _, queue := range aq.queues {
|
||||
if queue.timer != nil {
|
||||
queue.timer.Stop()
|
||||
}
|
||||
}
|
||||
clear(aq.queues)
|
||||
}
|
||||
aq.mu.Unlock()
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
aq.drainWG.Wait()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("%w: %v", ErrAnnounceQueueDrainTimeout, ctx.Err())
|
||||
}
|
||||
}
|
||||
|
||||
// drain merges items into a single announce message and calls onDrain.
|
||||
func (aq *AnnounceQueue) drain(sessionKey string, items []AnnounceQueueItem, meta AnnounceMetadata) {
|
||||
if len(items) == 0 || aq.onDrain == nil {
|
||||
@@ -147,11 +211,11 @@ func FormatBatchedAnnounce(items []AnnounceQueueItem, roster SubagentRoster) str
|
||||
replyInstruction := BuildReplyInstruction(roster)
|
||||
|
||||
return fmt.Sprintf(
|
||||
"[System Message] A subagent task %q just %s.\n\n"+
|
||||
"[System Message] A subagent task %q just %s (task=%s, parent=%s, depth=%d).\n\n"+
|
||||
"Result:\n%s\n\n"+
|
||||
"Stats: runtime %s, iterations %d, tokens %d in / %d out\n\n"+
|
||||
"%s",
|
||||
item.Label, statusLabel, item.Result,
|
||||
item.Label, statusLabel, item.SubagentID, item.ParentTaskID, item.Depth, item.Result,
|
||||
item.Runtime.Round(time.Millisecond), item.Iterations,
|
||||
item.InputTokens, item.OutputTokens,
|
||||
replyInstruction,
|
||||
@@ -171,8 +235,8 @@ func FormatBatchedAnnounce(items []AnnounceQueueItem, roster SubagentRoster) str
|
||||
}
|
||||
|
||||
sb.WriteString(fmt.Sprintf(
|
||||
"\n---\nTask #%d: %q %s (runtime %s, iterations %d, tokens %d/%d)\nResult: %s\n",
|
||||
i+1, item.Label, statusLabel,
|
||||
"\n---\nTask #%d: %q %s (task=%s, parent=%s, depth=%d, runtime %s, iterations %d, tokens %d/%d)\nResult: %s\n",
|
||||
i+1, item.Label, statusLabel, item.SubagentID, item.ParentTaskID, item.Depth,
|
||||
item.Runtime.Round(time.Millisecond), item.Iterations,
|
||||
item.InputTokens, item.OutputTokens,
|
||||
item.Result,
|
||||
@@ -188,11 +252,12 @@ func FormatBatchedAnnounce(items []AnnounceQueueItem, roster SubagentRoster) str
|
||||
// buildReplyInstruction generates the instruction block for the parent LLM,
|
||||
// including a deterministic roster of all subagent tasks with their statuses.
|
||||
func BuildReplyInstruction(roster SubagentRoster) string {
|
||||
// Count running tasks from roster
|
||||
running := 0
|
||||
for _, e := range roster.Entries {
|
||||
if e.Status == TaskStatusRunning {
|
||||
running++
|
||||
running := roster.Active
|
||||
if running == 0 {
|
||||
for _, entry := range roster.Entries {
|
||||
if entry.Status == TaskStatusRunning {
|
||||
running++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -200,8 +265,10 @@ func BuildReplyInstruction(roster SubagentRoster) string {
|
||||
var rosterBlock string
|
||||
if len(roster.Entries) > 0 {
|
||||
var sb strings.Builder
|
||||
sb.WriteString(fmt.Sprintf("Subagent roster (%d spawned, max %d per agent):\n",
|
||||
roster.Total, roster.MaxPerAgent))
|
||||
sb.WriteString(fmt.Sprintf(
|
||||
"Subagent roster (%d active / %d max; %d retained total):\n",
|
||||
running, roster.MaxPerAgent, roster.Total,
|
||||
))
|
||||
for _, e := range roster.Entries {
|
||||
sb.WriteString(fmt.Sprintf(" [%-9s] %s\n", e.Status, e.Label))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestAnnounceQueueCloseDropsPendingDebounce(t *testing.T) {
|
||||
drained := make(chan struct{}, 1)
|
||||
queue := NewAnnounceQueue(20, 20, func(string, []AnnounceQueueItem, AnnounceMetadata) {
|
||||
drained <- struct{}{}
|
||||
})
|
||||
queue.Enqueue("session", AnnounceQueueItem{SubagentID: "task"}, AnnounceMetadata{})
|
||||
if err := queue.CloseContext(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-drained:
|
||||
t.Fatal("pending announce drained after queue close")
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnnounceQueueCloseContextBoundsActiveDrain(t *testing.T) {
|
||||
started := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
queue := NewAnnounceQueue(1, 1, func(string, []AnnounceQueueItem, AnnounceMetadata) {
|
||||
close(started)
|
||||
<-release
|
||||
})
|
||||
queue.Enqueue("session", AnnounceQueueItem{SubagentID: "task"}, AnnounceMetadata{})
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("announce drain did not start")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
||||
defer cancel()
|
||||
err := queue.CloseContext(ctx)
|
||||
if !errors.Is(err, ErrAnnounceQueueDrainTimeout) {
|
||||
t.Fatalf("CloseContext error = %v, want typed drain timeout", err)
|
||||
}
|
||||
close(release)
|
||||
if err := queue.CloseContext(context.Background()); err != nil {
|
||||
t.Fatalf("CloseContext retry: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/bus"
|
||||
)
|
||||
|
||||
const asyncCompletionAttempts = 3
|
||||
|
||||
const (
|
||||
asyncTerminalPersistenceAttempts = 12
|
||||
asyncPersistenceAttemptTimeout = 2 * time.Second
|
||||
asyncPersistenceMaxBackoff = time.Second
|
||||
)
|
||||
|
||||
// PublishAsyncCompletion retries only after child execution has released its
|
||||
// admission lease. The durable task row remains the fallback if every attempt
|
||||
// finds the inbound queue saturated.
|
||||
func PublishAsyncCompletion(
|
||||
ctx context.Context,
|
||||
msgBus *bus.MessageBus,
|
||||
message bus.InboundMessage,
|
||||
) bool {
|
||||
if msgBus == nil {
|
||||
return true
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
for attempt := range asyncCompletionAttempts {
|
||||
if msgBus.TryPublishInbound(message) {
|
||||
return true
|
||||
}
|
||||
if attempt == asyncCompletionAttempts-1 {
|
||||
break
|
||||
}
|
||||
timer := time.NewTimer(time.Duration(attempt+1) * 10 * time.Millisecond)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return false
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func retryAsyncPersistence(
|
||||
ctx context.Context,
|
||||
operation func(context.Context) error,
|
||||
) error {
|
||||
return retryPersistence(ctx, asyncCompletionAttempts, operation)
|
||||
}
|
||||
|
||||
func retryTerminalPersistence(
|
||||
ctx context.Context,
|
||||
operation func(context.Context) error,
|
||||
) error {
|
||||
return retryPersistence(ctx, asyncTerminalPersistenceAttempts, operation)
|
||||
}
|
||||
|
||||
func retryPersistence(
|
||||
ctx context.Context,
|
||||
attempts int,
|
||||
operation func(context.Context) error,
|
||||
) error {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
var err error
|
||||
for attempt := range attempts {
|
||||
attemptCtx, cancel := context.WithTimeout(ctx, asyncPersistenceAttemptTimeout)
|
||||
err = operation(attemptCtx)
|
||||
cancel()
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if attempt == attempts-1 {
|
||||
break
|
||||
}
|
||||
backoff := min(time.Duration(1<<attempt)*25*time.Millisecond, asyncPersistenceMaxBackoff)
|
||||
timer := time.NewTimer(backoff)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return err
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,337 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/bus"
|
||||
orchestration "github.com/nextlevelbuilder/goclaw/internal/childrun"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
)
|
||||
|
||||
func TestSpawnAsyncReturnsDurableCompletionAndGetSurvivesManagerStateLoss(t *testing.T) {
|
||||
manager := NewSubagentManager(
|
||||
&recordingSubagentProvider{response: "durable spawn result"},
|
||||
nil,
|
||||
"model",
|
||||
nil,
|
||||
NewRegistry,
|
||||
SubagentConfig{MaxConcurrent: 2, MaxSpawnDepth: 2, MaxChildrenPerAgent: 2},
|
||||
)
|
||||
taskStore := newRecordingSubagentTaskStore()
|
||||
manager.SetTaskStore(taskStore)
|
||||
tool := NewSpawnTool(manager, "parent", 0)
|
||||
ctx := subagentTestContext("parent")
|
||||
|
||||
accepted := tool.Execute(ctx, map[string]any{"task": "persist this", "mode": "async"})
|
||||
if accepted == nil || accepted.IsError {
|
||||
t.Fatalf("spawn result = %#v", accepted)
|
||||
}
|
||||
var receipt struct {
|
||||
CompletionID string `json:"completion_id"`
|
||||
TaskID string `json:"task_id"`
|
||||
}
|
||||
if err := json.NewDecoder(strings.NewReader(accepted.ForLLM)).Decode(&receipt); err != nil {
|
||||
t.Fatalf("decode accepted result: %v\n%s", err, accepted.ForLLM)
|
||||
}
|
||||
completionID, err := uuid.Parse(receipt.CompletionID)
|
||||
if err != nil || receipt.TaskID == "" {
|
||||
t.Fatalf("receipt = %#v, parse error = %v", receipt, err)
|
||||
}
|
||||
manager.Close()
|
||||
|
||||
// A restarted manager has no in-memory tasks but can still retrieve the row.
|
||||
restarted := NewSubagentManager(nil, nil, "", nil, nil, SubagentConfig{})
|
||||
restarted.SetTaskStore(taskStore)
|
||||
getTool := NewSpawnTool(restarted, "parent", 0)
|
||||
got := getTool.Execute(ctx, map[string]any{
|
||||
"action": "get",
|
||||
"completion_id": completionID.String(),
|
||||
})
|
||||
if got == nil || got.IsError || !strings.Contains(got.ForLLM, "durable spawn result") {
|
||||
t.Fatalf("durable get result = %#v", got)
|
||||
}
|
||||
|
||||
otherRoot := store.WithAgentID(ctx, uuid.New())
|
||||
if result := getTool.Execute(otherRoot, map[string]any{
|
||||
"action": "get",
|
||||
"completion_id": completionID.String(),
|
||||
}); result == nil || !result.IsError {
|
||||
t.Fatalf("other root retrieved completion: %#v", result)
|
||||
}
|
||||
|
||||
restarted.Close()
|
||||
}
|
||||
|
||||
func TestSpawnAsyncRejectsAcceptanceWhenDurableCreateFails(t *testing.T) {
|
||||
provider := &recordingSubagentProvider{}
|
||||
manager := NewSubagentManager(
|
||||
provider,
|
||||
nil,
|
||||
"model",
|
||||
nil,
|
||||
NewRegistry,
|
||||
SubagentConfig{MaxConcurrent: 1, MaxSpawnDepth: 1, MaxChildrenPerAgent: 1},
|
||||
)
|
||||
taskStore := newRecordingSubagentTaskStore()
|
||||
taskStore.createErr = errors.New("database unavailable")
|
||||
manager.SetTaskStore(taskStore)
|
||||
|
||||
_, err := manager.Spawn(
|
||||
subagentTestContext("parent"),
|
||||
"parent",
|
||||
0,
|
||||
"must not run",
|
||||
"durable",
|
||||
"",
|
||||
"test",
|
||||
"chat",
|
||||
"",
|
||||
nil,
|
||||
)
|
||||
if err == nil || !strings.Contains(err.Error(), "persist accepted subagent") {
|
||||
t.Fatalf("spawn error = %v", err)
|
||||
}
|
||||
if provider.calls != 0 {
|
||||
t.Fatalf("provider calls = %d, want 0", provider.calls)
|
||||
}
|
||||
manager.Close()
|
||||
}
|
||||
|
||||
func TestSpawnAsyncDoesNotLabelAnnouncementDeliveredBeforeTerminalPersistence(t *testing.T) {
|
||||
messageBus := bus.New()
|
||||
manager := NewSubagentManager(
|
||||
&recordingSubagentProvider{response: "terminal result"},
|
||||
nil,
|
||||
"model",
|
||||
messageBus,
|
||||
NewRegistry,
|
||||
SubagentConfig{MaxConcurrent: 1, MaxSpawnDepth: 1, MaxChildrenPerAgent: 1},
|
||||
)
|
||||
taskStore := newRecordingSubagentTaskStore()
|
||||
taskStore.updateErr = errors.New("terminal database write failed")
|
||||
manager.SetTaskStore(taskStore)
|
||||
|
||||
_, err := manager.Spawn(
|
||||
subagentTestContext("parent"),
|
||||
"parent",
|
||||
0,
|
||||
"complete but fail persistence",
|
||||
"durability-ordering",
|
||||
"",
|
||||
"test",
|
||||
"chat",
|
||||
"",
|
||||
nil,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("spawn: %v", err)
|
||||
}
|
||||
manager.Close()
|
||||
|
||||
select {
|
||||
case metadata := <-taskStore.metadata:
|
||||
t.Fatalf("announcement metadata recorded before terminal persistence: %#v", metadata)
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelegateAsyncDurableGetIsSourceAgentScoped(t *testing.T) {
|
||||
taskStore := newRecordingSubagentTaskStore()
|
||||
tool := newDelegateTestTool(t, noopAgentLink{}, func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
|
||||
return DelegateResult{Content: "durable delegation result"}, nil
|
||||
})
|
||||
tool.SetTaskStore(taskStore)
|
||||
ctx := makeDelegateCtx(t)
|
||||
|
||||
accepted := tool.Execute(ctx, map[string]any{
|
||||
"agent_key": "child-agent",
|
||||
"task": "persist delegation",
|
||||
"mode": "async",
|
||||
})
|
||||
if accepted == nil || accepted.IsError {
|
||||
t.Fatalf("delegate result = %#v", accepted)
|
||||
}
|
||||
var receipt struct {
|
||||
DelegationID string `json:"delegation_id"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(accepted.ForLLM), &receipt); err != nil {
|
||||
t.Fatalf("decode delegation receipt: %v", err)
|
||||
}
|
||||
if _, err := uuid.Parse(receipt.DelegationID); err != nil {
|
||||
t.Fatalf("delegation id = %q: %v", receipt.DelegationID, err)
|
||||
}
|
||||
tool.Close()
|
||||
select {
|
||||
case metadata := <-taskStore.metadata:
|
||||
if metadata[asyncCompletionDeliveryKey] != asyncCompletionDeliveryMissed {
|
||||
t.Fatalf("noninteractive delegate announcement metadata = %#v, want undelivered", metadata)
|
||||
}
|
||||
default:
|
||||
t.Fatal("noninteractive delegate did not record get-only fallback")
|
||||
}
|
||||
delegationID := uuid.MustParse(receipt.DelegationID)
|
||||
if err := taskStore.UpdateMetadata(
|
||||
ctx,
|
||||
store.AgentIDFromContext(ctx),
|
||||
delegationID,
|
||||
map[string]any{
|
||||
asyncCompletionMediaKey: []persistedCompletionMedia{{
|
||||
Path: ".delegations/" + receipt.DelegationID + "/report.pdf",
|
||||
MimeType: "application/pdf",
|
||||
Filename: "report.pdf",
|
||||
}},
|
||||
},
|
||||
); err != nil {
|
||||
t.Fatalf("persist logical completion media: %v", err)
|
||||
}
|
||||
|
||||
restarted := NewDelegateTool(nil, nil, nil, nil)
|
||||
restarted.SetTaskStore(taskStore)
|
||||
defer restarted.Close()
|
||||
got := restarted.Execute(ctx, map[string]any{
|
||||
"action": "get",
|
||||
"delegation_id": receipt.DelegationID,
|
||||
})
|
||||
if got == nil || got.IsError ||
|
||||
!strings.Contains(got.ForLLM, "durable delegation result") ||
|
||||
!strings.Contains(got.ForLLM, ".delegations/"+receipt.DelegationID+"/report.pdf") {
|
||||
t.Fatalf("durable delegate get = %#v", got)
|
||||
}
|
||||
|
||||
otherAgent := store.WithAgentID(ctx, uuid.New())
|
||||
if result := restarted.Execute(otherAgent, map[string]any{
|
||||
"action": "get",
|
||||
"delegation_id": receipt.DelegationID,
|
||||
}); result == nil || !result.IsError {
|
||||
t.Fatalf("other agent retrieved delegation: %#v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelegateAsyncRunningPersistenceDoesNotRetainAdmissionPermit(t *testing.T) {
|
||||
taskStore := newRecordingSubagentTaskStore()
|
||||
var runningAttempts atomic.Int32
|
||||
runningDeadline := make(chan time.Duration, 1)
|
||||
persistenceStarted := make(chan struct{}, 1)
|
||||
persistenceRelease := make(chan struct{})
|
||||
defer func() {
|
||||
select {
|
||||
case <-persistenceRelease:
|
||||
default:
|
||||
close(persistenceRelease)
|
||||
}
|
||||
}()
|
||||
taskStore.updateHook = func(ctx context.Context, status string) error {
|
||||
if status != TaskStatusRunning {
|
||||
return nil
|
||||
}
|
||||
if runningAttempts.Add(1) != 1 {
|
||||
return nil
|
||||
}
|
||||
deadline, ok := ctx.Deadline()
|
||||
if !ok {
|
||||
t.Fatal("running persistence has no deadline")
|
||||
}
|
||||
runningDeadline <- time.Until(deadline)
|
||||
persistenceStarted <- struct{}{}
|
||||
<-persistenceRelease
|
||||
return errors.New("transient running persistence failure")
|
||||
}
|
||||
|
||||
admission := orchestration.NewChildRunAdmission(1, 4)
|
||||
runStarted := make(chan string, 2)
|
||||
tool := NewDelegateToolWithAdmission(noopAgentLink{}, noopAgentCRUD{}, nil, func(_ context.Context, req DelegateRequest) (DelegateResult, error) {
|
||||
runStarted <- req.Task
|
||||
return DelegateResult{Content: "done"}, nil
|
||||
}, admission)
|
||||
tool.SetWorkspace(t.TempDir())
|
||||
t.Cleanup(tool.Close)
|
||||
tool.SetTaskStore(taskStore)
|
||||
|
||||
first := tool.Execute(makeDelegateCtx(t), map[string]any{
|
||||
"agent_key": "child-agent",
|
||||
"task": "first",
|
||||
"mode": "async",
|
||||
})
|
||||
if first == nil || first.IsError {
|
||||
t.Fatalf("first delegate result = %#v", first)
|
||||
}
|
||||
select {
|
||||
case task := <-runStarted:
|
||||
if task != "first" {
|
||||
t.Fatalf("first started task = %q", task)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("first delegate run did not start")
|
||||
}
|
||||
select {
|
||||
case <-persistenceStarted:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("running persistence did not start")
|
||||
}
|
||||
|
||||
second := tool.Execute(makeDelegateCtx(t), map[string]any{
|
||||
"agent_key": "child-agent",
|
||||
"task": "second",
|
||||
"mode": "async",
|
||||
})
|
||||
if second == nil || second.IsError {
|
||||
t.Fatalf("second delegate result = %#v", second)
|
||||
}
|
||||
select {
|
||||
case task := <-runStarted:
|
||||
if task != "second" {
|
||||
t.Fatalf("second started task = %q", task)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("running-status persistence retained the only admission permit")
|
||||
}
|
||||
close(persistenceRelease)
|
||||
tool.Close()
|
||||
|
||||
select {
|
||||
case remaining := <-runningDeadline:
|
||||
if remaining <= 0 || remaining > 500*time.Millisecond {
|
||||
t.Fatalf("running persistence deadline = %s, want at most 500ms", remaining)
|
||||
}
|
||||
default:
|
||||
t.Fatal("running persistence was not attempted")
|
||||
}
|
||||
if attempts := runningAttempts.Load(); attempts != 2 {
|
||||
t.Fatalf("running persistence attempts = %d, want one per accepted delegation", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompletionMediaDescriptorsNeverPersistHostPaths(t *testing.T) {
|
||||
workspace := t.TempDir()
|
||||
inside := filepath.Join(workspace, ".uploads", "report.pdf")
|
||||
outside := filepath.Join(t.TempDir(), "secret.txt")
|
||||
descriptors := completionMediaDescriptors([]bus.MediaFile{
|
||||
{Path: inside, MimeType: "application/pdf", Filename: "report.pdf"},
|
||||
{Path: outside, MimeType: "text/plain", Filename: "secret.txt"},
|
||||
}, workspace, "")
|
||||
|
||||
if len(descriptors) != 1 || descriptors[0].Path != ".uploads/report.pdf" {
|
||||
t.Fatalf("completion media = %#v, want one logical workspace path", descriptors)
|
||||
}
|
||||
encoded, err := json.Marshal(descriptors)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(encoded), workspace) || strings.Contains(string(encoded), outside) {
|
||||
t.Fatalf("completion media leaked host path: %s", encoded)
|
||||
}
|
||||
if payload := persistedCompletionMediaPayload([]map[string]any{{
|
||||
"path": outside,
|
||||
}}); len(payload) != 0 {
|
||||
t.Fatalf("absolute persisted media was returned: %#v", payload)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/bus"
|
||||
)
|
||||
|
||||
const asyncCompletionMediaKey = "completion_media"
|
||||
|
||||
type persistedCompletionMedia struct {
|
||||
Path string `json:"path"`
|
||||
MimeType string `json:"mime_type,omitempty"`
|
||||
Filename string `json:"filename,omitempty"`
|
||||
Caption string `json:"caption,omitempty"`
|
||||
}
|
||||
|
||||
// completionMediaDescriptors converts runtime host paths into workspace-safe
|
||||
// logical paths. Outside-workspace media is intentionally omitted.
|
||||
func completionMediaDescriptors(
|
||||
media []bus.MediaFile,
|
||||
workspace, logicalPrefix string,
|
||||
) []persistedCompletionMedia {
|
||||
if len(media) == 0 || workspace == "" {
|
||||
return nil
|
||||
}
|
||||
descriptors := make([]persistedCompletionMedia, 0, len(media))
|
||||
for _, item := range media {
|
||||
rawPath := item.Path
|
||||
if !filepath.IsAbs(rawPath) {
|
||||
rawPath = filepath.Join(workspace, rawPath)
|
||||
}
|
||||
absolutePath, err := filepath.Abs(rawPath)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
relativePath, err := filepath.Rel(workspace, absolutePath)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
logicalPath := filepath.ToSlash(relativePath)
|
||||
if logicalPrefix != "" {
|
||||
logicalPath = strings.TrimSuffix(logicalPrefix, "/") + "/" + logicalPath
|
||||
}
|
||||
normalized, err := validateArtifactRelativePath(logicalPath)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
descriptors = append(descriptors, persistedCompletionMedia{
|
||||
Path: normalized,
|
||||
MimeType: item.MimeType,
|
||||
Filename: item.Filename,
|
||||
Caption: item.Caption,
|
||||
})
|
||||
}
|
||||
return descriptors
|
||||
}
|
||||
|
||||
func persistedCompletionMediaPayload(raw any) []persistedCompletionMedia {
|
||||
encoded, err := json.Marshal(raw)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var decoded []persistedCompletionMedia
|
||||
if err := json.Unmarshal(encoded, &decoded); err != nil {
|
||||
return nil
|
||||
}
|
||||
safe := decoded[:0]
|
||||
for _, item := range decoded {
|
||||
normalized, err := validateArtifactRelativePath(item.Path)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
item.Path = normalized
|
||||
safe = append(safe, item)
|
||||
}
|
||||
return safe
|
||||
}
|
||||
@@ -2,12 +2,14 @@ package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"sync"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
orchestration "github.com/nextlevelbuilder/goclaw/internal/childrun"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/config"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/sandbox"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
@@ -33,9 +35,76 @@ const (
|
||||
ctxAgentPolicy toolContextKey = "tool_agent_policy" // per-agent tool policy for MCP bridge enforcement
|
||||
ctxSessionKey toolContextKey = "tool_session_key" // origin session key for announce routing
|
||||
ctxRunKind toolContextKey = "tool_run_kind" // "notification", "announce", "delegation"
|
||||
ctxSubagentScope toolContextKey = "subagent_task_scope"
|
||||
ctxSubagentTaskID toolContextKey = "subagent_task_id"
|
||||
ctxSubagentDepth toolContextKey = "subagent_depth"
|
||||
ctxChildRunLease toolContextKey = "child_run_lease"
|
||||
ctxTelegramManagerPermissions toolContextKey = "telegram_manager_permissions"
|
||||
)
|
||||
|
||||
func withSubagentExecution(ctx context.Context, scope TaskScope, taskID string, depth int, lease *orchestration.ChildRunLease) context.Context {
|
||||
ctx = context.WithValue(ctx, ctxSubagentScope, scope)
|
||||
ctx = context.WithValue(ctx, ctxSubagentTaskID, taskID)
|
||||
ctx = context.WithValue(ctx, ctxSubagentDepth, depth)
|
||||
return context.WithValue(ctx, ctxChildRunLease, lease)
|
||||
}
|
||||
|
||||
func childRunLeaseFromContext(ctx context.Context) *orchestration.ChildRunLease {
|
||||
lease, _ := ctx.Value(ctxChildRunLease).(*orchestration.ChildRunLease)
|
||||
return lease
|
||||
}
|
||||
|
||||
// withDelegatedAgentExecution starts a fresh semantic spawn tree for the target
|
||||
// agent while retaining the admission lease used by synchronous continuations.
|
||||
func withDelegatedAgentExecution(ctx context.Context, lease *orchestration.ChildRunLease) context.Context {
|
||||
ctx = context.WithValue(ctx, ctxSubagentScope, TaskScope{})
|
||||
ctx = context.WithValue(ctx, ctxSubagentTaskID, "")
|
||||
ctx = context.WithValue(ctx, ctxSubagentDepth, 0)
|
||||
ctx = WithSubagentConfig(ctx, nil)
|
||||
return context.WithValue(ctx, ctxChildRunLease, lease)
|
||||
}
|
||||
|
||||
func subagentScopeFromContext(ctx context.Context) TaskScope {
|
||||
if scope, ok := ctx.Value(ctxSubagentScope).(TaskScope); ok {
|
||||
if scope.TenantID != uuid.Nil || scope.RootAgentID != uuid.Nil || scope.RootAgentKey != "" {
|
||||
return scope
|
||||
}
|
||||
}
|
||||
return TaskScope{
|
||||
TenantID: store.TenantIDFromContext(ctx),
|
||||
RootAgentID: store.AgentIDFromContext(ctx),
|
||||
RootAgentKey: ToolAgentKeyFromCtx(ctx),
|
||||
}
|
||||
}
|
||||
|
||||
func subagentTaskIDFromContext(ctx context.Context) string {
|
||||
id, _ := ctx.Value(ctxSubagentTaskID).(string)
|
||||
return id
|
||||
}
|
||||
|
||||
func subagentDepthFromContext(ctx context.Context, fallback int) int {
|
||||
if depth, ok := ctx.Value(ctxSubagentDepth).(int); ok {
|
||||
return depth
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
// childRunContinuationLineage returns structural admission lineage. It is
|
||||
// intentionally separate from semantic sub-agent depth, which resets at every
|
||||
// Agent Link delegation boundary.
|
||||
func childRunContinuationLineage(
|
||||
ctx context.Context,
|
||||
fallbackParentTaskID string,
|
||||
fallbackDepth int,
|
||||
) (string, int) {
|
||||
if lease := childRunLeaseFromContext(ctx); lease != nil {
|
||||
if parentTaskID, parentDepth, ok := lease.ContinuationParent(); ok {
|
||||
return parentTaskID, parentDepth + 1
|
||||
}
|
||||
}
|
||||
return fallbackParentTaskID, fallbackDepth
|
||||
}
|
||||
|
||||
// ctxRateLimitOverride carries a per-agent tool rate limit (calls/hour) that
|
||||
// overrides the global tools.rate_limit_per_hour. 0 means "use the global".
|
||||
const ctxRateLimitOverride toolContextKey = "tool_rate_limit_override"
|
||||
@@ -53,6 +122,13 @@ type MediaPathLoader interface {
|
||||
LoadPath(id string) (string, error)
|
||||
}
|
||||
|
||||
// MediaPathRootProvider optionally exposes the managed root that authorizes
|
||||
// legacy media returned by MediaPathLoader. Implementations must return a
|
||||
// stable root; callers still validate containment and regular-file semantics.
|
||||
type MediaPathRootProvider interface {
|
||||
MediaRootPath() string
|
||||
}
|
||||
|
||||
func WithToolChannel(ctx context.Context, channel string) context.Context {
|
||||
return context.WithValue(ctx, ctxChannel, channel)
|
||||
}
|
||||
@@ -137,7 +213,7 @@ func WithToolWorkspace(ctx context.Context, ws string) context.Context {
|
||||
}
|
||||
|
||||
func ToolWorkspaceFromCtx(ctx context.Context) string {
|
||||
if v, _ := ctx.Value(ctxWorkspace).(string); v != "" {
|
||||
if v, ok := ctx.Value(ctxWorkspace).(string); ok {
|
||||
return v
|
||||
}
|
||||
if rc := store.RunContextFromCtx(ctx); rc != nil {
|
||||
@@ -374,7 +450,8 @@ func WithSubagentConfig(ctx context.Context, cfg *config.SubagentsConfig) contex
|
||||
}
|
||||
|
||||
func SubagentConfigFromCtx(ctx context.Context) *config.SubagentsConfig {
|
||||
if v, _ := ctx.Value(ctxSubagentCfg).(*config.SubagentsConfig); v != nil {
|
||||
if raw := ctx.Value(ctxSubagentCfg); raw != nil {
|
||||
v, _ := raw.(*config.SubagentsConfig)
|
||||
return v
|
||||
}
|
||||
if rc := store.RunContextFromCtx(ctx); rc != nil {
|
||||
@@ -432,7 +509,7 @@ func WithToolTeamID(ctx context.Context, teamID string) context.Context {
|
||||
|
||||
// ToolTeamIDFromCtx returns the dispatching team's ID from context.
|
||||
func ToolTeamIDFromCtx(ctx context.Context) string {
|
||||
if v, _ := ctx.Value(ctxTeamID).(string); v != "" {
|
||||
if v, ok := ctx.Value(ctxTeamID).(string); ok {
|
||||
return v
|
||||
}
|
||||
if rc := store.RunContextFromCtx(ctx); rc != nil {
|
||||
@@ -453,7 +530,7 @@ func WithToolTeamWorkspace(ctx context.Context, dir string) context.Context {
|
||||
|
||||
// ToolTeamWorkspaceFromCtx returns the team shared workspace directory path.
|
||||
func ToolTeamWorkspaceFromCtx(ctx context.Context) string {
|
||||
if v, _ := ctx.Value(ctxTeamWorkspace).(string); v != "" {
|
||||
if v, ok := ctx.Value(ctxTeamWorkspace).(string); ok {
|
||||
return v
|
||||
}
|
||||
if rc := store.RunContextFromCtx(ctx); rc != nil {
|
||||
@@ -495,7 +572,7 @@ func WithTeamTaskID(ctx context.Context, taskID string) context.Context {
|
||||
|
||||
// TeamTaskIDFromCtx returns the delegation's team task ID from context.
|
||||
func TeamTaskIDFromCtx(ctx context.Context) string {
|
||||
if v, _ := ctx.Value(ctxTeamTaskID).(string); v != "" {
|
||||
if v, ok := ctx.Value(ctxTeamTaskID).(string); ok {
|
||||
return v
|
||||
}
|
||||
if rc := store.RunContextFromCtx(ctx); rc != nil {
|
||||
@@ -518,7 +595,7 @@ func WithDelegationID(ctx context.Context, id string) context.Context {
|
||||
// DelegationIDFromCtx returns the active delegation ID. Falls back to
|
||||
// RunContext when no explicit context key is present.
|
||||
func DelegationIDFromCtx(ctx context.Context) string {
|
||||
if v, _ := ctx.Value(ctxDelegationID).(string); v != "" {
|
||||
if v, ok := ctx.Value(ctxDelegationID).(string); ok {
|
||||
return v
|
||||
}
|
||||
if rc := store.RunContextFromCtx(ctx); rc != nil {
|
||||
@@ -539,7 +616,7 @@ func WithLeaderAgentID(ctx context.Context, id string) context.Context {
|
||||
|
||||
// LeaderAgentIDFromCtx returns the leader's agent UUID string from context.
|
||||
func LeaderAgentIDFromCtx(ctx context.Context) string {
|
||||
if v, _ := ctx.Value(ctxLeaderAgentID).(string); v != "" {
|
||||
if v, ok := ctx.Value(ctxLeaderAgentID).(string); ok {
|
||||
return v
|
||||
}
|
||||
if rc := store.RunContextFromCtx(ctx); rc != nil {
|
||||
@@ -820,7 +897,7 @@ func WithTenantAllowedPaths(ctx context.Context, paths []string) context.Context
|
||||
// TenantAllowedPathsFromCtx returns tenant-specific allowed paths from context.
|
||||
// Falls back to RunContext for subagent inheritance.
|
||||
func TenantAllowedPathsFromCtx(ctx context.Context) []string {
|
||||
if v, _ := ctx.Value(ctxTenantAllowedPaths).([]string); len(v) > 0 {
|
||||
if v, ok := ctx.Value(ctxTenantAllowedPaths).([]string); ok {
|
||||
return v
|
||||
}
|
||||
if rc := store.RunContextFromCtx(ctx); rc != nil {
|
||||
@@ -828,3 +905,31 @@ func TenantAllowedPathsFromCtx(ctx context.Context) []string {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
const ctxDelegationArtifactInputs toolContextKey = "tool_delegation_artifact_inputs"
|
||||
|
||||
// WithDelegationArtifactInputs installs the runtime-only host root backing the
|
||||
// logical read-only inputs/ alias for an admitted Agent Link run.
|
||||
func WithDelegationArtifactInputs(ctx context.Context, root string) context.Context {
|
||||
return context.WithValue(ctx, ctxDelegationArtifactInputs, root)
|
||||
}
|
||||
|
||||
// DelegationArtifactInputsFromCtx returns the runtime-only staged-input root.
|
||||
// It is deliberately not copied into RunContext, prompts, traces, or results.
|
||||
func DelegationArtifactInputsFromCtx(ctx context.Context) string {
|
||||
root, _ := ctx.Value(ctxDelegationArtifactInputs).(string)
|
||||
return root
|
||||
}
|
||||
|
||||
// IsDelegationArtifactRun reports whether filesystem/media tools are executing
|
||||
// inside an isolated Agent Link artifact exchange.
|
||||
func IsDelegationArtifactRun(ctx context.Context) bool {
|
||||
return DelegationIDFromCtx(ctx) != "" && DelegationArtifactInputsFromCtx(ctx) != ""
|
||||
}
|
||||
|
||||
func validateDelegationChildRunMode(ctx context.Context, operation, mode string) error {
|
||||
if IsDelegationArtifactRun(ctx) && mode != "sync" {
|
||||
return fmt.Errorf("%s mode %q is not allowed inside an Agent Link artifact run; use mode=\"sync\"", operation, mode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -78,7 +79,11 @@ func (t *CreateImageTool) resolveReferenceImages(ctx context.Context, args map[s
|
||||
}
|
||||
seenPaths[path] = true
|
||||
|
||||
ext := strings.ToLower(filepath.Ext(path))
|
||||
resolved, err := resolveStructuredMediaPath(ctx, path, "reference image")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ext := strings.ToLower(filepath.Ext(resolved))
|
||||
mimeTypes := map[string]string{
|
||||
".jpg": "image/jpeg", ".jpeg": "image/jpeg",
|
||||
".png": "image/png", ".gif": "image/gif",
|
||||
@@ -88,14 +93,6 @@ func (t *CreateImageTool) resolveReferenceImages(ctx context.Context, args map[s
|
||||
if !ok {
|
||||
mime = "image/png"
|
||||
}
|
||||
workspace := ToolWorkspaceFromCtx(ctx)
|
||||
resolved, err := resolvePathWithAllowed(path, workspace, effectiveRestrict(ctx, true), allowedWithTeamWorkspace(ctx, nil))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid reference image path: %w", err)
|
||||
}
|
||||
if err := checkDeniedPath(resolved, workspace, nil); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data, err := os.ReadFile(resolved)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read reference image file: %w", err)
|
||||
@@ -136,18 +133,25 @@ func (t *CreateImageTool) resolveReferenceImages(ctx context.Context, args map[s
|
||||
seenIDs[id] = true
|
||||
|
||||
images := MediaImagesFromCtx(ctx)
|
||||
if len(images) == 0 {
|
||||
return nil, fmt.Errorf("no images available in conversation context")
|
||||
}
|
||||
var img providers.ImageContent
|
||||
if id == "latest" {
|
||||
ref, refErr := resolveImageMediaRef(ctx, id)
|
||||
if refErr == nil && ref.Path != "" {
|
||||
fileImages, err := (&ReadImageTool{}).loadImageFromPath(ctx, ref.Path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to resolve reference image media_id %q: %w", id, err)
|
||||
}
|
||||
img = fileImages[0]
|
||||
} else if id == "latest" && len(images) > 0 {
|
||||
img = images[len(images)-1]
|
||||
} else {
|
||||
var idx int
|
||||
if _, err := fmt.Sscanf(id, "%d", &idx); err == nil && idx >= 0 && idx < len(images) {
|
||||
idx, err := strconv.Atoi(id)
|
||||
if err == nil && idx >= 0 && idx < len(images) {
|
||||
img = images[idx]
|
||||
} else {
|
||||
img = images[len(images)-1]
|
||||
if refErr != nil {
|
||||
return nil, refErr
|
||||
}
|
||||
return nil, fmt.Errorf("image media_id %q has no accessible workspace path", id)
|
||||
}
|
||||
}
|
||||
dataBytes, _ := base64.StdEncoding.DecodeString(img.Data)
|
||||
@@ -226,9 +230,9 @@ func (t *CreateImageTool) Parameters() map[string]any {
|
||||
"items": map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"path": map[string]any{"type": "string", "description": "Workspace file path to a reference image."},
|
||||
"path": map[string]any{"type": "string", "description": "Logical workspace path to a reference image, such as .uploads/photo.jpg or delegation input inputs/photo.jpg. Never guess an absolute path."},
|
||||
"url": map[string]any{"type": "string", "description": "HTTP/HTTPS URL of a reference image."},
|
||||
"id": map[string]any{"type": "string", "description": "Media ID of a reference image from the chat."},
|
||||
"id": map[string]any{"type": "string", "description": "Exact image ID from a <media:image id=\"...\"> tag, or 'latest', for in-process tool execution. Claude CLI/MCP callers should use path."},
|
||||
"strength": map[string]any{"type": "number", "description": "Reference strength (0.0 to 1.0) specific to this image."},
|
||||
"description": map[string]any{"type": "string", "description": "Description of the role or content of this reference image (e.g. 'Lâm', 'Quân')."},
|
||||
},
|
||||
|
||||
@@ -396,6 +396,59 @@ func TestCreateImageTool_ResolveReferenceImage_ID(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateImageTool_ResolveReferenceImage_ExactID(t *testing.T) {
|
||||
workspace := t.TempDir()
|
||||
firstPath := filepath.Join(workspace, "first.png")
|
||||
secondPath := filepath.Join(workspace, "second.png")
|
||||
firstData := []byte("first-image")
|
||||
secondData := []byte("second-image")
|
||||
if err := os.WriteFile(firstPath, firstData, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(secondPath, secondData, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ctx := WithToolWorkspace(context.Background(), workspace)
|
||||
ctx = WithMediaImageRefs(ctx, []providers.MediaRef{
|
||||
{ID: "image-first", Kind: "image", MimeType: "image/png", Path: firstPath},
|
||||
{ID: "image-second", Kind: "image", MimeType: "image/png", Path: secondPath},
|
||||
})
|
||||
|
||||
got, err := NewCreateImageTool(nil).resolveReferenceImages(ctx, map[string]any{
|
||||
"ref_images": []any{map[string]any{"id": "image-first"}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("resolve exact media ID: %v", err)
|
||||
}
|
||||
if len(got) != 1 || string(got[0].Data) != string(firstData) {
|
||||
t.Fatalf("resolved images = %#v, want exact first image", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateImageTool_ResolveReferenceImage_UnknownIDFails(t *testing.T) {
|
||||
workspace := t.TempDir()
|
||||
imagePath := filepath.Join(workspace, "known.png")
|
||||
if err := os.WriteFile(imagePath, []byte("known-image"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := WithToolWorkspace(context.Background(), workspace)
|
||||
ctx = WithMediaImages(ctx, []providers.ImageContent{{
|
||||
MimeType: "image/png",
|
||||
Data: base64.StdEncoding.EncodeToString([]byte("fallback-must-not-be-used")),
|
||||
}})
|
||||
ctx = WithMediaImageRefs(ctx, []providers.MediaRef{{
|
||||
ID: "known-id", Kind: "image", MimeType: "image/png", Path: imagePath,
|
||||
}})
|
||||
|
||||
_, err := NewCreateImageTool(nil).resolveReferenceImages(ctx, map[string]any{
|
||||
"ref_images": []any{map[string]any{"id": "missing-id"}},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), `media_id "missing-id" not found`) {
|
||||
t.Fatalf("unknown media ID error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateImageTool_MultipleReferenceImages_MixedSources(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
refFile := filepath.Join(tmpDir, "ref.png")
|
||||
|
||||
@@ -727,7 +727,7 @@ func (t *ExecTool) executeCredentialedSandbox(ctx context.Context, absPath strin
|
||||
return ErrorResult(fmt.Sprintf("credentialed sandbox path mapping: %v", cwdErr))
|
||||
}
|
||||
|
||||
sb, err := t.sandboxMgr.Get(ctx, sandboxKey, mountWorkspace, SandboxConfigFromCtx(ctx))
|
||||
sb, err := acquireToolSandbox(ctx, t.sandboxMgr, sandboxKey, mountWorkspace)
|
||||
if err != nil {
|
||||
slog.Warn("security.credentialed_exec_sandbox_unavailable",
|
||||
"binary", absPath, "error", err)
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
)
|
||||
|
||||
const delegateCompletionTargetKey = "target_agent_key"
|
||||
|
||||
const delegateRunningPersistenceTimeout = 250 * time.Millisecond
|
||||
|
||||
func (t *DelegateTool) createDelegateCompletion(ctx context.Context, req DelegateRequest) error {
|
||||
if t.taskStore == nil {
|
||||
return nil
|
||||
}
|
||||
delegationID, err := uuid.Parse(req.DelegationID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid delegation ID: %w", err)
|
||||
}
|
||||
tenantID := parseUUIDOrNil(req.TenantID)
|
||||
if tenantID == uuid.Nil || req.FromAgentID == uuid.Nil {
|
||||
return fmt.Errorf("delegation completion requires tenant and source agent")
|
||||
}
|
||||
var sessionKey, originChannel, originChatID, originPeerKind, originUserID *string
|
||||
if req.SessionKey != "" {
|
||||
sessionKey = &req.SessionKey
|
||||
}
|
||||
if req.Channel != "" {
|
||||
originChannel = &req.Channel
|
||||
}
|
||||
if req.ChatID != "" {
|
||||
originChatID = &req.ChatID
|
||||
}
|
||||
if req.PeerKind != "" {
|
||||
originPeerKind = &req.PeerKind
|
||||
}
|
||||
if req.UserID != "" {
|
||||
originUserID = &req.UserID
|
||||
}
|
||||
data := &store.SubagentTaskData{
|
||||
BaseModel: store.BaseModel{ID: delegationID},
|
||||
TenantID: tenantID,
|
||||
RootAgentID: req.FromAgentID,
|
||||
ParentAgentKey: req.FromAgentKey,
|
||||
SessionKey: sessionKey,
|
||||
Subject: "Delegate to " + req.ToAgentKey,
|
||||
Description: req.Task,
|
||||
Status: TaskStatusQueued,
|
||||
Depth: subagentDepthFromContext(ctx, 0) + 1,
|
||||
OriginChannel: originChannel,
|
||||
OriginChatID: originChatID,
|
||||
OriginPeerKind: originPeerKind,
|
||||
OriginUserID: originUserID,
|
||||
Metadata: map[string]any{
|
||||
asyncCompletionKindKey: asyncCompletionKindDelegate,
|
||||
delegateCompletionTargetKey: req.ToAgentKey,
|
||||
asyncCompletionDeliveryKey: asyncCompletionDeliveryPending,
|
||||
},
|
||||
}
|
||||
dbCtx := store.WithTenantID(context.Background(), tenantID)
|
||||
if err := retryAsyncPersistence(dbCtx, func(attemptCtx context.Context) error {
|
||||
return t.taskStore.Create(attemptCtx, data)
|
||||
}); err != nil {
|
||||
return fmt.Errorf("persist accepted delegation %s: %w", req.DelegationID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *DelegateTool) updateDelegateCompletion(
|
||||
req DelegateRequest,
|
||||
status string,
|
||||
result *string,
|
||||
) error {
|
||||
if t.taskStore == nil {
|
||||
return nil
|
||||
}
|
||||
delegationID := parseUUIDOrNil(req.DelegationID)
|
||||
tenantID := parseUUIDOrNil(req.TenantID)
|
||||
if delegationID == uuid.Nil || tenantID == uuid.Nil || req.FromAgentID == uuid.Nil {
|
||||
return fmt.Errorf("delegation completion requires valid tenant, source agent, and delegation ID")
|
||||
}
|
||||
dbCtx := store.WithTenantID(context.Background(), tenantID)
|
||||
if err := retryTerminalPersistence(dbCtx, func(attemptCtx context.Context) error {
|
||||
return t.taskStore.UpdateStatus(
|
||||
attemptCtx,
|
||||
req.FromAgentID,
|
||||
delegationID,
|
||||
status,
|
||||
result,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
)
|
||||
}); err != nil {
|
||||
slog.Error("delegate.async.completion_persist_failed",
|
||||
"delegation_id", req.DelegationID,
|
||||
"from_agent_id", req.FromAgentID,
|
||||
"status", status,
|
||||
"error", err,
|
||||
)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// updateDelegateRunning is observability-only: execution may proceed even when
|
||||
// this transition cannot be persisted. Keep it to one short attempt outside
|
||||
// the admission callback so a database outage cannot monopolize a child-run
|
||||
// permit.
|
||||
func (t *DelegateTool) updateDelegateRunning(req DelegateRequest) error {
|
||||
if t.taskStore == nil {
|
||||
return nil
|
||||
}
|
||||
delegationID := parseUUIDOrNil(req.DelegationID)
|
||||
tenantID := parseUUIDOrNil(req.TenantID)
|
||||
if delegationID == uuid.Nil || tenantID == uuid.Nil || req.FromAgentID == uuid.Nil {
|
||||
return fmt.Errorf("delegation completion requires valid tenant, source agent, and delegation ID")
|
||||
}
|
||||
dbCtx, cancel := context.WithTimeout(
|
||||
store.WithTenantID(context.Background(), tenantID),
|
||||
delegateRunningPersistenceTimeout,
|
||||
)
|
||||
defer cancel()
|
||||
err := t.taskStore.UpdateStatus(
|
||||
dbCtx,
|
||||
req.FromAgentID,
|
||||
delegationID,
|
||||
TaskStatusRunning,
|
||||
nil,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
)
|
||||
if err != nil {
|
||||
slog.Warn("delegate.async.running_persist_failed",
|
||||
"delegation_id", req.DelegationID,
|
||||
"from_agent_id", req.FromAgentID,
|
||||
"error", err,
|
||||
)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (t *DelegateTool) updateDelegateCompletionMedia(
|
||||
req DelegateRequest,
|
||||
media []persistedCompletionMedia,
|
||||
) error {
|
||||
if t.taskStore == nil || len(media) == 0 {
|
||||
return nil
|
||||
}
|
||||
delegationID := parseUUIDOrNil(req.DelegationID)
|
||||
tenantID := parseUUIDOrNil(req.TenantID)
|
||||
if delegationID == uuid.Nil || tenantID == uuid.Nil || req.FromAgentID == uuid.Nil {
|
||||
return fmt.Errorf("delegation completion requires valid tenant, source agent, and delegation ID")
|
||||
}
|
||||
dbCtx := store.WithTenantID(context.Background(), tenantID)
|
||||
if err := retryTerminalPersistence(dbCtx, func(attemptCtx context.Context) error {
|
||||
return t.taskStore.UpdateMetadata(attemptCtx, req.FromAgentID, delegationID, map[string]any{
|
||||
asyncCompletionMediaKey: media,
|
||||
})
|
||||
}); err != nil {
|
||||
slog.Error("delegate.async.completion_media_persist_failed",
|
||||
"delegation_id", req.DelegationID,
|
||||
"from_agent_id", req.FromAgentID,
|
||||
"error", err,
|
||||
)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *DelegateTool) updateDelegateAnnouncement(req DelegateRequest, delivered bool) {
|
||||
if t.taskStore == nil {
|
||||
return
|
||||
}
|
||||
delegationID := parseUUIDOrNil(req.DelegationID)
|
||||
tenantID := parseUUIDOrNil(req.TenantID)
|
||||
if delegationID == uuid.Nil || tenantID == uuid.Nil || req.FromAgentID == uuid.Nil {
|
||||
return
|
||||
}
|
||||
status := asyncCompletionDeliveryMissed
|
||||
if delivered {
|
||||
status = asyncCompletionDeliveryDone
|
||||
}
|
||||
dbCtx := store.WithTenantID(context.Background(), tenantID)
|
||||
if err := retryAsyncPersistence(dbCtx, func(attemptCtx context.Context) error {
|
||||
return t.taskStore.UpdateMetadata(attemptCtx, req.FromAgentID, delegationID, map[string]any{
|
||||
asyncCompletionDeliveryKey: status,
|
||||
})
|
||||
}); err != nil {
|
||||
slog.Warn("delegate.async.announcement_status_failed",
|
||||
"delegation_id", req.DelegationID,
|
||||
"error", err,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func (t *DelegateTool) executeGetCompletion(ctx context.Context, args map[string]any) *Result {
|
||||
rawID, _ := args["delegation_id"].(string)
|
||||
delegationID, err := uuid.Parse(rawID)
|
||||
if err != nil {
|
||||
return ErrorResult("delegation_id must be a valid UUID")
|
||||
}
|
||||
tenantID := store.TenantIDFromContext(ctx)
|
||||
fromAgentID := store.AgentIDFromContext(ctx)
|
||||
if tenantID == uuid.Nil || fromAgentID == uuid.Nil {
|
||||
return ErrorResult("delegate result lookup requires tenant and agent context")
|
||||
}
|
||||
if t.taskStore == nil {
|
||||
return ErrorResult("durable delegation task tracking is unavailable")
|
||||
}
|
||||
dbCtx := store.WithTenantID(context.WithoutCancel(ctx), tenantID)
|
||||
task, err := t.taskStore.Get(dbCtx, fromAgentID, delegationID)
|
||||
if err != nil {
|
||||
return ErrorResult("failed to load delegation result")
|
||||
}
|
||||
if task == nil || completionKind(task.Metadata) != asyncCompletionKindDelegate {
|
||||
return ErrorResult("delegation result not found")
|
||||
}
|
||||
payload := persistedCompletionPayload(task)
|
||||
payload["delegation_id"] = task.ID.String()
|
||||
delete(payload, "completion_id")
|
||||
if target, ok := task.Metadata[delegateCompletionTargetKey].(string); ok && target != "" {
|
||||
payload["agent"] = target
|
||||
}
|
||||
encoded, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return ErrorResult("failed to encode delegation result")
|
||||
}
|
||||
return NewResult(string(encoded))
|
||||
}
|
||||
+956
-97
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,708 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/config"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
)
|
||||
|
||||
const delegationArtifactManifestMaxBytes = 1024 * 1024
|
||||
|
||||
type recoveredPublicationStatus uint8
|
||||
|
||||
const (
|
||||
recoveredPublicationMissing recoveredPublicationStatus = iota
|
||||
recoveredPublicationInvalid
|
||||
recoveredPublicationValid
|
||||
)
|
||||
|
||||
func isArtifactSymlinkError(err error) bool {
|
||||
return errors.Is(err, errArtifactSymlink) || errors.Is(err, errArtifactReparsePoint)
|
||||
}
|
||||
|
||||
type retainedDelegationArtifact struct {
|
||||
tenantWorkspace string
|
||||
tenantID uuid.UUID
|
||||
tenantSlug string
|
||||
delegationID uuid.UUID
|
||||
retainUntil time.Time
|
||||
callerLocation *delegationArtifactCallerLocation
|
||||
publicationTempPath string
|
||||
publicationDurable bool
|
||||
onCleaned func()
|
||||
}
|
||||
|
||||
func (t *DelegateTool) resolveDelegationCallerLocation(
|
||||
job *delegateArtifactJob,
|
||||
) *delegationArtifactCallerLocation {
|
||||
type candidate struct {
|
||||
name string
|
||||
root string
|
||||
}
|
||||
candidates := []candidate{{name: "workspace", root: job.tenantWorkspace}}
|
||||
if t.dataDir != "" {
|
||||
candidates = append(candidates, candidate{
|
||||
name: "data",
|
||||
root: config.TenantDataDir(t.dataDir, job.tenantID, job.tenantSlug),
|
||||
})
|
||||
}
|
||||
for _, candidate := range candidates {
|
||||
relativePath, ok := artifactRelativeToRoot(candidate.root, job.callerWorkspace)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
return &delegationArtifactCallerLocation{
|
||||
Base: candidate.name,
|
||||
RelativePath: relativePath,
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func artifactRelativeToRoot(root, target string) (string, bool) {
|
||||
if root == "" || target == "" {
|
||||
return "", false
|
||||
}
|
||||
absoluteRoot, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
absoluteTarget, err := filepath.Abs(target)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
relativePath, err := filepath.Rel(absoluteRoot, absoluteTarget)
|
||||
if err != nil || relativePath == ".." ||
|
||||
strings.HasPrefix(relativePath, ".."+string(filepath.Separator)) {
|
||||
return "", false
|
||||
}
|
||||
relativePath = filepath.ToSlash(relativePath)
|
||||
if relativePath == "." {
|
||||
return relativePath, true
|
||||
}
|
||||
validated, err := validateArtifactRelativePath(relativePath)
|
||||
return validated, err == nil
|
||||
}
|
||||
|
||||
func (t *DelegateTool) updateActiveDelegationLifecycle(
|
||||
exchange *DelegationArtifactExchange,
|
||||
job *delegateArtifactJob,
|
||||
) error {
|
||||
state, err := readDelegationArtifactLifecycleState(exchange.root, job.delegationID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if state.Status != artifactLifecycleStaging {
|
||||
return ErrArtifactState
|
||||
}
|
||||
state.TenantSlug = job.tenantSlug
|
||||
state.CallerLocation = job.callerLocation
|
||||
return exchange.persistLifecycleState(state)
|
||||
}
|
||||
|
||||
func (t *DelegateTool) markDelegationRunning(
|
||||
exchange *DelegationArtifactExchange,
|
||||
job *delegateArtifactJob,
|
||||
now time.Time,
|
||||
) error {
|
||||
state, err := readDelegationArtifactLifecycleState(exchange.root, job.delegationID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if state.Status != artifactLifecycleStaging {
|
||||
return ErrArtifactState
|
||||
}
|
||||
state.TenantSlug = job.tenantSlug
|
||||
state.Status = artifactLifecycleRunning
|
||||
state.FailedAt = nil
|
||||
state.RetainUntil = now.UTC().Add(delegationArtifactFailureTTL)
|
||||
state.ReasonCode = "artifact_running"
|
||||
state.CallerLocation = job.callerLocation
|
||||
state.PublicationTempPath = ""
|
||||
return exchange.persistLifecycleState(state)
|
||||
}
|
||||
|
||||
func (t *DelegateTool) markDelegationPublishing(
|
||||
exchange *DelegationArtifactExchange,
|
||||
job *delegateArtifactJob,
|
||||
tempPath string,
|
||||
now time.Time,
|
||||
) error {
|
||||
if err := validateDelegationPublicationTempPath(job.delegationID, tempPath); err != nil {
|
||||
return err
|
||||
}
|
||||
state, err := readDelegationArtifactLifecycleState(exchange.root, job.delegationID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if state.Status != artifactLifecycleRunning {
|
||||
return ErrArtifactState
|
||||
}
|
||||
state.TenantSlug = job.tenantSlug
|
||||
state.Status = artifactLifecyclePublishing
|
||||
state.FailedAt = nil
|
||||
state.RetainUntil = now.UTC().Add(delegationArtifactFailureTTL)
|
||||
state.ReasonCode = "artifact_publishing"
|
||||
state.CallerLocation = job.callerLocation
|
||||
state.PublicationTempPath = tempPath
|
||||
return exchange.persistLifecycleState(state)
|
||||
}
|
||||
|
||||
func (t *DelegateTool) markDelegationPublished(
|
||||
exchange *DelegationArtifactExchange,
|
||||
job *delegateArtifactJob,
|
||||
now time.Time,
|
||||
) error {
|
||||
state, err := readDelegationArtifactLifecycleState(exchange.root, job.delegationID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if state.Status != artifactLifecyclePublishing {
|
||||
return ErrArtifactState
|
||||
}
|
||||
state.Status = artifactLifecyclePublished
|
||||
state.FailedAt = nil
|
||||
state.RetainUntil = now.UTC()
|
||||
state.ReasonCode = "artifact_published"
|
||||
return exchange.persistLifecycleState(state)
|
||||
}
|
||||
|
||||
func (t *DelegateTool) registerRetainedDelegationExchange(
|
||||
exchange *DelegationArtifactExchange,
|
||||
job *delegateArtifactJob,
|
||||
status delegationArtifactLifecycleStatus,
|
||||
) error {
|
||||
if exchange == nil || job == nil {
|
||||
return ErrArtifactState
|
||||
}
|
||||
|
||||
retention, ok := exchange.FailureRetention()
|
||||
if !ok {
|
||||
return ErrArtifactState
|
||||
}
|
||||
state, err := readDelegationArtifactLifecycleState(exchange.root, job.delegationID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
failedAt := retention.FailedAt.UTC()
|
||||
state.TenantSlug = job.tenantSlug
|
||||
if status != artifactLifecycleCancelled {
|
||||
status = artifactLifecycleFailed
|
||||
}
|
||||
state.Status = status
|
||||
state.FailedAt = &failedAt
|
||||
state.RetainUntil = retention.RetainUntil.UTC()
|
||||
state.ReasonCode = retention.ReasonCode
|
||||
state.CallerLocation = job.callerLocation
|
||||
if err := exchange.persistLifecycleState(state); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
t.addRetainedDelegationArtifact(retainedDelegationArtifact{
|
||||
tenantWorkspace: job.tenantWorkspace,
|
||||
tenantID: job.tenantID,
|
||||
tenantSlug: job.tenantSlug,
|
||||
delegationID: job.delegationID,
|
||||
retainUntil: retention.RetainUntil,
|
||||
callerLocation: job.callerLocation,
|
||||
publicationTempPath: state.PublicationTempPath,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *DelegateTool) registerPublishedDelegationCleanup(
|
||||
job *delegateArtifactJob,
|
||||
publicationTempPath string,
|
||||
onCleaned func(),
|
||||
) {
|
||||
t.addRetainedDelegationArtifact(retainedDelegationArtifact{
|
||||
tenantWorkspace: job.tenantWorkspace,
|
||||
tenantID: job.tenantID,
|
||||
tenantSlug: job.tenantSlug,
|
||||
delegationID: job.delegationID,
|
||||
retainUntil: time.Time{},
|
||||
callerLocation: job.callerLocation,
|
||||
publicationTempPath: publicationTempPath,
|
||||
publicationDurable: true,
|
||||
onCleaned: onCleaned,
|
||||
})
|
||||
}
|
||||
|
||||
func (t *DelegateTool) addRetainedDelegationArtifact(item retainedDelegationArtifact) {
|
||||
key := retainedDelegationArtifactKey(item.tenantWorkspace, item.delegationID)
|
||||
t.retainedMu.Lock()
|
||||
defer t.retainedMu.Unlock()
|
||||
if t.sweeperClosed {
|
||||
return
|
||||
}
|
||||
if existing, ok := t.retained[key]; ok && item.onCleaned == nil {
|
||||
item.onCleaned = existing.onCleaned
|
||||
}
|
||||
t.retained[key] = item
|
||||
if t.sweeperStarted {
|
||||
return
|
||||
}
|
||||
t.sweeperStarted = true
|
||||
go t.runDelegationArtifactSweeper()
|
||||
}
|
||||
|
||||
func retainedDelegationArtifactKey(tenantWorkspace string, delegationID uuid.UUID) string {
|
||||
return tenantWorkspace + "\x00" + delegationID.String()
|
||||
}
|
||||
|
||||
func (t *DelegateTool) runDelegationArtifactSweeper() {
|
||||
ticker := time.NewTicker(delegationArtifactSweepInterval)
|
||||
defer func() {
|
||||
ticker.Stop()
|
||||
close(t.sweeperDone)
|
||||
}()
|
||||
for {
|
||||
select {
|
||||
case now := <-ticker.C:
|
||||
t.recoverRetainedDelegationExchanges()
|
||||
t.sweepRetainedDelegationExchanges(now)
|
||||
case <-t.sweeperStop:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (t *DelegateTool) recoverRetainedDelegationExchanges() {
|
||||
for _, tenantWorkspace := range t.delegationTenantWorkspaces() {
|
||||
if err := t.recoverTenantDelegationExchanges(tenantWorkspace); err != nil {
|
||||
slog.Warn("delegate.artifact_recovery_scan_failed")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (t *DelegateTool) delegationTenantWorkspaces() []string {
|
||||
if t.workspace == "" {
|
||||
return nil
|
||||
}
|
||||
roots := make([]string, 0, 8)
|
||||
if info, err := os.Lstat(t.workspace); err == nil && info.IsDir() && info.Mode()&os.ModeSymlink == 0 {
|
||||
roots = append(roots, t.workspace)
|
||||
}
|
||||
tenantParent := filepath.Join(t.workspace, "tenants")
|
||||
entries, err := os.ReadDir(tenantParent)
|
||||
if err != nil {
|
||||
return roots
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if entry.Type()&os.ModeSymlink != 0 {
|
||||
continue
|
||||
}
|
||||
info, err := entry.Info()
|
||||
if err != nil || !info.IsDir() {
|
||||
continue
|
||||
}
|
||||
roots = append(roots, filepath.Join(tenantParent, entry.Name()))
|
||||
}
|
||||
sort.Strings(roots)
|
||||
return roots
|
||||
}
|
||||
|
||||
func (t *DelegateTool) recoverTenantDelegationExchanges(tenantWorkspace string) error {
|
||||
tenantRoot, err := openArtifactSecureRoot(tenantWorkspace)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tenantRoot.close()
|
||||
delegationsRoot, err := tenantRoot.openSubroot("collaboration/delegations")
|
||||
if err != nil {
|
||||
if isArtifactNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
defer delegationsRoot.close()
|
||||
names, err := tenantRoot.readDir("collaboration/delegations")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sort.Strings(names)
|
||||
recoveredAt := time.Now().UTC()
|
||||
for _, name := range names {
|
||||
delegationID, err := uuid.Parse(name)
|
||||
if err != nil || delegationID == uuid.Nil || delegationID.String() != name {
|
||||
if cleanupErr := delegationsRoot.removeTree(name); cleanupErr != nil {
|
||||
slog.Warn("delegate.artifact_invalid_exchange_cleanup_failed",
|
||||
"entry", name,
|
||||
"error", cleanupErr,
|
||||
)
|
||||
} else {
|
||||
slog.Warn("security.delegation_artifact_invalid_exchange_removed",
|
||||
"entry", name,
|
||||
)
|
||||
}
|
||||
continue
|
||||
}
|
||||
exchangeRoot, err := delegationsRoot.openSubroot(name)
|
||||
if err != nil {
|
||||
slog.Warn("delegate.artifact_exchange_recovery_open_failed",
|
||||
"delegation_id", delegationID,
|
||||
"error", err,
|
||||
)
|
||||
continue
|
||||
}
|
||||
state, stateErr := readDelegationArtifactLifecycleState(exchangeRoot, delegationID)
|
||||
if stateErr != nil {
|
||||
_ = exchangeRoot.close()
|
||||
t.addRetainedDelegationArtifact(retainedDelegationArtifact{
|
||||
tenantWorkspace: tenantWorkspace,
|
||||
delegationID: delegationID,
|
||||
retainUntil: recoveredAt.Add(delegationArtifactFailureTTL),
|
||||
})
|
||||
slog.Warn("delegate.artifact_corrupt_exchange_retained",
|
||||
"delegation_id", delegationID,
|
||||
"error", stateErr,
|
||||
)
|
||||
continue
|
||||
}
|
||||
tenantID, err := uuid.Parse(state.TenantID)
|
||||
if err != nil || tenantID == uuid.Nil {
|
||||
_ = exchangeRoot.close()
|
||||
continue
|
||||
}
|
||||
if !t.lifecycleStateMatchesTenantWorkspace(state, tenantWorkspace, tenantID) {
|
||||
_ = exchangeRoot.close()
|
||||
continue
|
||||
}
|
||||
retainUntil := state.RetainUntil
|
||||
publicationDurable := false
|
||||
switch state.Status {
|
||||
case artifactLifecyclePublishing:
|
||||
item := retainedDelegationArtifact{
|
||||
tenantWorkspace: tenantWorkspace,
|
||||
tenantID: tenantID,
|
||||
tenantSlug: state.TenantSlug,
|
||||
delegationID: delegationID,
|
||||
retainUntil: state.RetainUntil,
|
||||
callerLocation: state.CallerLocation,
|
||||
publicationTempPath: state.PublicationTempPath,
|
||||
}
|
||||
publicationStatus, inspectErr := t.inspectRecoveredPublication(item)
|
||||
if inspectErr != nil {
|
||||
_ = exchangeRoot.close()
|
||||
t.addRetainedDelegationArtifact(item)
|
||||
slog.Warn("delegate.artifact_publication_recovery_deferred",
|
||||
"delegation_id", delegationID,
|
||||
"error", inspectErr,
|
||||
)
|
||||
continue
|
||||
}
|
||||
if publicationStatus == recoveredPublicationValid {
|
||||
state.Status = artifactLifecyclePublished
|
||||
state.FailedAt = nil
|
||||
state.RetainUntil = recoveredAt
|
||||
state.ReasonCode = "artifact_published"
|
||||
retainUntil = time.Time{}
|
||||
publicationDurable = true
|
||||
if err := persistDelegationArtifactLifecycleState(exchangeRoot, delegationID, state); err != nil {
|
||||
slog.Warn("delegate.artifact_publication_recovery_state_failed",
|
||||
"delegation_id", delegationID,
|
||||
"error", err,
|
||||
)
|
||||
}
|
||||
break
|
||||
}
|
||||
fallthrough
|
||||
case artifactLifecycleStaging, artifactLifecycleRunning:
|
||||
failedAt := recoveredAt
|
||||
state.Status = artifactLifecycleFailed
|
||||
state.FailedAt = &failedAt
|
||||
state.RetainUntil = recoveredAt.Add(delegationArtifactFailureTTL)
|
||||
state.ReasonCode = "artifact_recovered_stale"
|
||||
if err := persistDelegationArtifactLifecycleState(exchangeRoot, delegationID, state); err != nil {
|
||||
_ = exchangeRoot.close()
|
||||
continue
|
||||
}
|
||||
retainUntil = state.RetainUntil
|
||||
case artifactLifecyclePublished:
|
||||
retainUntil = time.Time{}
|
||||
publicationDurable = true
|
||||
}
|
||||
_ = exchangeRoot.close()
|
||||
t.addRetainedDelegationArtifact(retainedDelegationArtifact{
|
||||
tenantWorkspace: tenantWorkspace,
|
||||
tenantID: tenantID,
|
||||
tenantSlug: state.TenantSlug,
|
||||
delegationID: delegationID,
|
||||
retainUntil: retainUntil,
|
||||
callerLocation: state.CallerLocation,
|
||||
publicationTempPath: state.PublicationTempPath,
|
||||
publicationDurable: publicationDurable,
|
||||
})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *DelegateTool) inspectRecoveredPublication(
|
||||
item retainedDelegationArtifact,
|
||||
) (recoveredPublicationStatus, error) {
|
||||
callerRootPath, err := t.resolveRetainedCallerRoot(item)
|
||||
if err != nil {
|
||||
return recoveredPublicationInvalid, nil
|
||||
}
|
||||
callerRoot, err := openArtifactSecureRoot(callerRootPath)
|
||||
if err != nil {
|
||||
if isArtifactNotExist(err) {
|
||||
return recoveredPublicationMissing, nil
|
||||
}
|
||||
return recoveredPublicationMissing, err
|
||||
}
|
||||
defer callerRoot.close()
|
||||
|
||||
finalPath := path.Join(".delegations", item.delegationID.String())
|
||||
publicationRoot, err := callerRoot.openSubroot(finalPath)
|
||||
if err != nil {
|
||||
switch {
|
||||
case isArtifactNotExist(err):
|
||||
return recoveredPublicationMissing, nil
|
||||
case isArtifactSymlinkError(err):
|
||||
return recoveredPublicationInvalid, nil
|
||||
default:
|
||||
return recoveredPublicationMissing, err
|
||||
}
|
||||
}
|
||||
defer publicationRoot.close()
|
||||
|
||||
entry, err := publicationRoot.openEntry("manifest.json")
|
||||
if err != nil {
|
||||
if isArtifactNotExist(err) || isArtifactSymlinkError(err) {
|
||||
return recoveredPublicationInvalid, nil
|
||||
}
|
||||
return recoveredPublicationMissing, err
|
||||
}
|
||||
defer entry.close()
|
||||
if entry.kind != artifactEntryRegular || entry.links != 1 ||
|
||||
entry.size <= 0 || entry.size > delegationArtifactManifestMaxBytes {
|
||||
return recoveredPublicationInvalid, nil
|
||||
}
|
||||
encoded, err := io.ReadAll(io.LimitReader(entry.file, delegationArtifactManifestMaxBytes+1))
|
||||
if err != nil {
|
||||
return recoveredPublicationMissing, err
|
||||
}
|
||||
if len(encoded) > delegationArtifactManifestMaxBytes {
|
||||
return recoveredPublicationInvalid, nil
|
||||
}
|
||||
var manifest DelegationArtifactManifest
|
||||
if err := json.Unmarshal(encoded, &manifest); err != nil {
|
||||
return recoveredPublicationInvalid, nil
|
||||
}
|
||||
if manifest.SchemaVersion != DelegationArtifactManifestVersion ||
|
||||
manifest.DelegationID != item.delegationID.String() ||
|
||||
manifest.OutputCount != len(manifest.Outputs) ||
|
||||
manifest.OutputCount < 0 ||
|
||||
manifest.OutputCount > DelegationArtifactMaxFiles ||
|
||||
manifest.OutputBytes < 0 ||
|
||||
manifest.OutputBytes > DelegationArtifactMaxTotalBytes {
|
||||
return recoveredPublicationInvalid, nil
|
||||
}
|
||||
|
||||
seen := make(map[string]struct{}, len(manifest.Outputs))
|
||||
var totalBytes int64
|
||||
for _, output := range manifest.Outputs {
|
||||
if err := validateManifestOutputPath(output.Path); err != nil ||
|
||||
output.SizeBytes < 0 ||
|
||||
output.SizeBytes > DelegationArtifactMaxFileBytes ||
|
||||
output.SHA256 == "" {
|
||||
return recoveredPublicationInvalid, nil
|
||||
}
|
||||
if _, exists := seen[output.Path]; exists {
|
||||
return recoveredPublicationInvalid, nil
|
||||
}
|
||||
seen[output.Path] = struct{}{}
|
||||
outputEntry, err := publicationRoot.openEntry(output.Path)
|
||||
if err != nil {
|
||||
if isArtifactNotExist(err) || isArtifactSymlinkError(err) {
|
||||
return recoveredPublicationInvalid, nil
|
||||
}
|
||||
return recoveredPublicationMissing, err
|
||||
}
|
||||
valid := outputEntry.kind == artifactEntryRegular &&
|
||||
outputEntry.links == 1 &&
|
||||
outputEntry.size == output.SizeBytes
|
||||
_ = outputEntry.close()
|
||||
if !valid {
|
||||
return recoveredPublicationInvalid, nil
|
||||
}
|
||||
totalBytes += output.SizeBytes
|
||||
if totalBytes > DelegationArtifactMaxTotalBytes {
|
||||
return recoveredPublicationInvalid, nil
|
||||
}
|
||||
}
|
||||
if totalBytes != manifest.OutputBytes {
|
||||
return recoveredPublicationInvalid, nil
|
||||
}
|
||||
return recoveredPublicationValid, nil
|
||||
}
|
||||
|
||||
func (t *DelegateTool) lifecycleStateMatchesTenantWorkspace(
|
||||
state delegationArtifactLifecycleState,
|
||||
tenantWorkspace string,
|
||||
tenantID uuid.UUID,
|
||||
) bool {
|
||||
if tenantID == store.MasterTenantID {
|
||||
expected, err := filepath.Abs(t.workspace)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
actual, err := filepath.Abs(tenantWorkspace)
|
||||
return err == nil && actual == expected
|
||||
}
|
||||
if state.TenantSlug == "" {
|
||||
// A staging state can be created immediately before its tenant slug is
|
||||
// added. It has no caller publication path and can only delete its scanned
|
||||
// exchange directory.
|
||||
return state.Status == artifactLifecycleStaging &&
|
||||
state.CallerLocation == nil &&
|
||||
state.PublicationTempPath == ""
|
||||
}
|
||||
expected := config.TenantWorkspace(t.workspace, tenantID, state.TenantSlug)
|
||||
expectedAbs, err := filepath.Abs(expected)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
actualAbs, err := filepath.Abs(tenantWorkspace)
|
||||
return err == nil && actualAbs == expectedAbs
|
||||
}
|
||||
|
||||
func (t *DelegateTool) sweepRetainedDelegationExchanges(now time.Time) {
|
||||
due := make([]retainedDelegationArtifact, 0, delegationArtifactSweepBatch)
|
||||
t.retainedMu.Lock()
|
||||
for _, item := range t.retained {
|
||||
if !now.Before(item.retainUntil) {
|
||||
due = append(due, item)
|
||||
if len(due) == delegationArtifactSweepBatch {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
t.retainedMu.Unlock()
|
||||
|
||||
for _, item := range due {
|
||||
if err := t.cleanupRetainedDelegationArtifact(item); err != nil {
|
||||
slog.Warn("delegate.artifact_retention_cleanup_failed")
|
||||
continue
|
||||
}
|
||||
t.retainedMu.Lock()
|
||||
delete(t.retained, retainedDelegationArtifactKey(item.tenantWorkspace, item.delegationID))
|
||||
t.retainedMu.Unlock()
|
||||
if item.onCleaned != nil {
|
||||
item.onCleaned()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (t *DelegateTool) cleanupRetainedDelegationArtifact(item retainedDelegationArtifact) error {
|
||||
if item.publicationTempPath != "" && !item.publicationDurable {
|
||||
if item.callerLocation == nil {
|
||||
return ErrArtifactState
|
||||
}
|
||||
if err := validateDelegationPublicationTempPath(item.delegationID, item.publicationTempPath); err != nil {
|
||||
return err
|
||||
}
|
||||
callerRootPath, err := t.resolveRetainedCallerRoot(item)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
callerRoot, err := openArtifactSecureRoot(callerRootPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = callerRoot.removeTree(item.publicationTempPath)
|
||||
_ = callerRoot.close()
|
||||
if err != nil && !isArtifactNotExist(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return t.tryRemoveDelegationExchange(item.tenantWorkspace, item.delegationID)
|
||||
}
|
||||
|
||||
func (t *DelegateTool) resolveRetainedCallerRoot(item retainedDelegationArtifact) (string, error) {
|
||||
if item.callerLocation == nil {
|
||||
return "", ErrArtifactState
|
||||
}
|
||||
var base string
|
||||
switch item.callerLocation.Base {
|
||||
case "workspace":
|
||||
base = item.tenantWorkspace
|
||||
case "data":
|
||||
if t.dataDir == "" {
|
||||
return "", ErrArtifactState
|
||||
}
|
||||
base = config.TenantDataDir(t.dataDir, item.tenantID, item.tenantSlug)
|
||||
default:
|
||||
return "", ErrArtifactState
|
||||
}
|
||||
relativePath := item.callerLocation.RelativePath
|
||||
if relativePath == "." {
|
||||
return base, nil
|
||||
}
|
||||
validated, err := validateArtifactRelativePath(relativePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filepath.Join(base, filepath.FromSlash(validated)), nil
|
||||
}
|
||||
|
||||
func validateDelegationPublicationTempPath(delegationID uuid.UUID, tempPath string) error {
|
||||
validated, err := validateArtifactRelativePath(tempPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
prefix := ".tmp-" + delegationID.String() + "-"
|
||||
if path.Dir(validated) != ".delegations" ||
|
||||
!strings.HasPrefix(path.Base(validated), prefix) {
|
||||
return ErrArtifactInvalidPath
|
||||
}
|
||||
suffix := strings.TrimPrefix(path.Base(validated), prefix)
|
||||
parsed, err := uuid.Parse(suffix)
|
||||
if err != nil || parsed == uuid.Nil || parsed.String() != suffix {
|
||||
return ErrArtifactInvalidPath
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *DelegateTool) removeDelegationExchange(
|
||||
tenantWorkspace string,
|
||||
delegationID uuid.UUID,
|
||||
) error {
|
||||
if tenantWorkspace == "" || delegationID == uuid.Nil {
|
||||
return ErrArtifactState
|
||||
}
|
||||
tenantRoot, err := openArtifactSecureRoot(tenantWorkspace)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tenantRoot.close()
|
||||
relativePath := path.Join("collaboration", "delegations", delegationID.String())
|
||||
if err := tenantRoot.removeTree(relativePath); err != nil && !isArtifactNotExist(err) {
|
||||
return fmt.Errorf("remove delegation exchange: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *DelegateTool) tryRemoveDelegationExchange(
|
||||
tenantWorkspace string,
|
||||
delegationID uuid.UUID,
|
||||
) error {
|
||||
if t.removeExchange != nil {
|
||||
return t.removeExchange(tenantWorkspace, delegationID)
|
||||
}
|
||||
return t.removeDelegationExchange(tenantWorkspace, delegationID)
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// resolveDelegationInputPath maps the logical inputs/ alias to the captured
|
||||
// exchange input directory. The host root never needs to be disclosed to the
|
||||
// delegated model.
|
||||
func resolveDelegationInputPath(ctx context.Context, raw string) (string, bool, error) {
|
||||
root := DelegationArtifactInputsFromCtx(ctx)
|
||||
if root == "" || filepath.IsAbs(raw) {
|
||||
return "", false, nil
|
||||
}
|
||||
logical := path.Clean(filepath.ToSlash(strings.TrimSpace(raw)))
|
||||
if logical == "inputs" {
|
||||
return filepath.Clean(root), true, nil
|
||||
}
|
||||
if !strings.HasPrefix(logical, "inputs/") {
|
||||
return "", false, nil
|
||||
}
|
||||
relative, err := validateArtifactRelativePath(strings.TrimPrefix(logical, "inputs/"))
|
||||
if err != nil {
|
||||
return "", true, err
|
||||
}
|
||||
resolved, err := resolvePathWithAllowed(
|
||||
filepath.FromSlash(relative),
|
||||
root,
|
||||
true,
|
||||
nil,
|
||||
)
|
||||
if err != nil {
|
||||
return "", true, fmt.Errorf("cannot access delegation input")
|
||||
}
|
||||
return resolved, true, nil
|
||||
}
|
||||
|
||||
func rejectDelegationInputMutation(ctx context.Context, raw string) error {
|
||||
if _, handled, err := resolveDelegationInputPath(ctx, raw); handled {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("delegation inputs are read-only")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// resolveStructuredMediaPath resolves logical delegation inputs for media
|
||||
// readers without widening filesystem-tool authority. Outside an artifact run
|
||||
// it preserves the normal workspace and Agent Team read rules.
|
||||
func resolveStructuredMediaPath(ctx context.Context, raw, kind string) (string, error) {
|
||||
return resolveStructuredMediaPathWithAllowed(ctx, raw, kind, allowedWithTeamWorkspace(ctx, nil))
|
||||
}
|
||||
|
||||
// resolveStructuredMediaRefPath resolves paths sourced from conversation
|
||||
// MediaRefs. Unlike explicit user/tool paths, a ref is authoritative only for
|
||||
// the active workspace or the current delegation's staged inputs.
|
||||
func resolveStructuredMediaRefPath(ctx context.Context, raw, kind string) (string, error) {
|
||||
if inputRoot := DelegationArtifactInputsFromCtx(ctx); inputRoot != "" && filepath.IsAbs(raw) {
|
||||
if resolved, err := resolvePathWithAllowed(raw, inputRoot, true, nil); err == nil {
|
||||
if err := ValidateRegularFileForRead(resolved); err != nil {
|
||||
return "", fmt.Errorf("%s delegation input is not a safe regular file", kind)
|
||||
}
|
||||
return filepath.Clean(resolved), nil
|
||||
}
|
||||
}
|
||||
return resolveStructuredMediaPathWithAllowed(ctx, raw, kind, nil)
|
||||
}
|
||||
|
||||
// resolveLoadedMediaRefPath preserves legacy media-store compatibility without
|
||||
// treating an arbitrary loader return value as filesystem authority.
|
||||
func resolveLoadedMediaRefPath(
|
||||
ctx context.Context,
|
||||
loader MediaPathLoader,
|
||||
raw, kind string,
|
||||
) (string, error) {
|
||||
if rooted, ok := loader.(MediaPathRootProvider); ok {
|
||||
root := rooted.MediaRootPath()
|
||||
if root != "" {
|
||||
if resolved, err := resolvePathWithAllowed(raw, root, true, nil); err == nil {
|
||||
if err := ValidateRegularFileForRead(resolved); err != nil {
|
||||
return "", fmt.Errorf("%s legacy media is not a safe regular file", kind)
|
||||
}
|
||||
return filepath.Clean(resolved), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return resolveStructuredMediaRefPath(ctx, raw, kind)
|
||||
}
|
||||
|
||||
func resolveStructuredMediaPathWithAllowed(ctx context.Context, raw, kind string, allowedPrefixes []string) (string, error) {
|
||||
if resolved, handled, err := resolveDelegationInputPath(ctx, raw); handled {
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("invalid %s delegation input path", kind)
|
||||
}
|
||||
if err := ValidateRegularFileForRead(resolved); err != nil {
|
||||
return "", fmt.Errorf("%s delegation input is not a safe regular file", kind)
|
||||
}
|
||||
return resolved, nil
|
||||
}
|
||||
|
||||
workspace := ToolWorkspaceFromCtx(ctx)
|
||||
resolved, err := resolvePathWithAllowed(raw, workspace, effectiveRestrict(ctx, true), allowedPrefixes)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("invalid %s path: %w", kind, err)
|
||||
}
|
||||
if err := checkDeniedPath(resolved, workspace, nil); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := ValidateRegularFileForRead(resolved); err != nil {
|
||||
return "", fmt.Errorf("%s path is not a safe regular file: %w", kind, err)
|
||||
}
|
||||
return filepath.Clean(resolved), nil
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/bus"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
)
|
||||
|
||||
func delegationArtifactToolContext(t *testing.T) (context.Context, string, string) {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
inputs := filepath.Join(root, "inputs")
|
||||
outputs := filepath.Join(root, "outputs")
|
||||
if err := os.MkdirAll(inputs, 0750); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.MkdirAll(outputs, 0750); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := WithDelegationID(context.Background(), uuid.NewString())
|
||||
ctx = WithDelegationArtifactInputs(ctx, inputs)
|
||||
ctx = WithToolWorkspace(ctx, outputs)
|
||||
return ctx, inputs, outputs
|
||||
}
|
||||
|
||||
func TestDelegationArtifactFilePolicy(t *testing.T) {
|
||||
ctx, inputs, outputs := delegationArtifactToolContext(t)
|
||||
if err := os.WriteFile(filepath.Join(inputs, "source.txt"), []byte("staged"), 0440); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
read := NewReadFileTool(outputs, true).Execute(ctx, map[string]any{"path": "inputs/source.txt"})
|
||||
if read.IsError || !strings.Contains(read.ForLLM, "staged") {
|
||||
t.Fatalf("read staged input = %#v", read)
|
||||
}
|
||||
listed := NewListFilesTool(outputs, true).Execute(ctx, map[string]any{"path": "inputs"})
|
||||
if listed.IsError || !strings.Contains(listed.ForLLM, "source.txt") {
|
||||
t.Fatalf("list staged inputs = %#v", listed)
|
||||
}
|
||||
|
||||
writeInput := NewWriteFileTool(outputs, true).Execute(ctx, map[string]any{
|
||||
"path": "inputs/source.txt", "content": "mutated",
|
||||
})
|
||||
if !writeInput.IsError || !strings.Contains(writeInput.ForLLM, "read-only") {
|
||||
t.Fatalf("input mutation = %#v, want read-only error", writeInput)
|
||||
}
|
||||
editInput := NewEditTool(outputs, true).Execute(ctx, map[string]any{
|
||||
"path": "inputs/source.txt", "old_string": "staged", "new_string": "mutated",
|
||||
})
|
||||
if !editInput.IsError || !strings.Contains(editInput.ForLLM, "read-only") {
|
||||
t.Fatalf("input edit = %#v, want read-only error", editInput)
|
||||
}
|
||||
|
||||
written := NewWriteFileTool(outputs, true).Execute(ctx, map[string]any{
|
||||
"path": "report.txt", "content": "result", "deliver": true,
|
||||
})
|
||||
if written.IsError || len(written.Media) != 0 {
|
||||
t.Fatalf("prepublication write delivery = %#v", written)
|
||||
}
|
||||
if got, err := os.ReadFile(filepath.Join(outputs, "report.txt")); err != nil || string(got) != "result" {
|
||||
t.Fatalf("output file = %q, %v", got, err)
|
||||
}
|
||||
|
||||
if result := NewSendFileTool(outputs, true).Execute(ctx, map[string]any{"path": "report.txt"}); !result.IsError {
|
||||
t.Fatalf("send_file published prepublication output: %#v", result)
|
||||
}
|
||||
if result := (&MessageTool{}).Execute(ctx, map[string]any{
|
||||
"action": "send", "message": "MEDIA:report.txt",
|
||||
}); !result.IsError {
|
||||
t.Fatalf("message published prepublication output: %#v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadDocumentArchiveUsesLogicalDelegationInputPath(t *testing.T) {
|
||||
ctx, inputs, outputs := delegationArtifactToolContext(t)
|
||||
archive := filepath.Join(inputs, "archive.zip")
|
||||
if err := os.WriteFile(archive, []byte("not a real zip"), 0440); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
result := NewReadDocumentTool(nil, nil).Execute(ctx, map[string]any{
|
||||
"path": "inputs/archive.zip",
|
||||
"prompt": "inspect archive",
|
||||
})
|
||||
if result.IsError {
|
||||
t.Fatalf("read_document archive = %#v", result)
|
||||
}
|
||||
if strings.Contains(result.ForLLM, inputs) || strings.Contains(result.ForLLM, outputs) {
|
||||
t.Fatalf("archive result leaked host path: %s", result.ForLLM)
|
||||
}
|
||||
if !strings.Contains(result.ForLLM, "inputs/archive.zip") {
|
||||
t.Fatalf("archive result omitted logical path: %s", result.ForLLM)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelegationArtifactDisablesVirtualMemoryRouting(t *testing.T) {
|
||||
ctx, _, outputs := delegationArtifactToolContext(t)
|
||||
agentID := uuid.New()
|
||||
ctx = store.WithAgentID(ctx, agentID)
|
||||
ctx = store.WithUserID(ctx, "user-1")
|
||||
memStore := newMockMemoryStore()
|
||||
memStore.docs[docKey(agentID.String(), "user-1", "MEMORY.md")] = "database-secret"
|
||||
interceptor := NewMemoryInterceptor(memStore, outputs)
|
||||
|
||||
if err := os.WriteFile(filepath.Join(outputs, "MEMORY.md"), []byte("physical-output"), 0640); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reader := NewReadFileTool(outputs, true)
|
||||
reader.SetMemoryInterceptor(interceptor)
|
||||
result := reader.Execute(ctx, map[string]any{"path": "MEMORY.md"})
|
||||
if result.IsError || strings.Contains(result.ForLLM, "database-secret") ||
|
||||
!strings.Contains(result.ForLLM, "physical-output") {
|
||||
t.Fatalf("delegation memory read routed virtually: %#v", result)
|
||||
}
|
||||
|
||||
writer := NewWriteFileTool(outputs, true)
|
||||
writer.SetMemoryInterceptor(interceptor)
|
||||
result = writer.Execute(ctx, map[string]any{"path": "MEMORY.md", "content": "new-output"})
|
||||
if result.IsError {
|
||||
t.Fatalf("physical MEMORY.md write failed: %#v", result)
|
||||
}
|
||||
if got := memStore.docs[docKey(agentID.String(), "user-1", "MEMORY.md")]; got != "database-secret" {
|
||||
t.Fatalf("virtual memory mutated: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelegationStructuredMediaPathsResolveOnlyStagedInputs(t *testing.T) {
|
||||
ctx, inputs, _ := delegationArtifactToolContext(t)
|
||||
staged := filepath.Join(inputs, "clip.mp4")
|
||||
if err := os.WriteFile(staged, []byte("video"), 0440); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := resolveStructuredMediaPath(ctx, "inputs/clip.mp4", "video")
|
||||
if err != nil {
|
||||
t.Fatalf("resolve staged media: %v", err)
|
||||
}
|
||||
want, err := filepath.EvalSymlinks(staged)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("resolved path = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
_, err = resolveStructuredMediaPath(ctx, "inputs/missing.mp4", "video")
|
||||
if err == nil {
|
||||
t.Fatal("missing staged media unexpectedly resolved")
|
||||
}
|
||||
if strings.Contains(err.Error(), filepath.Dir(inputs)) {
|
||||
t.Fatalf("error leaked exchange host root: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateImageReferenceUsesLogicalDelegationInputPath(t *testing.T) {
|
||||
ctx, inputs, _ := delegationArtifactToolContext(t)
|
||||
staged := filepath.Join(inputs, "reference.png")
|
||||
want := []byte("staged-reference")
|
||||
if err := os.WriteFile(staged, want, 0440); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
refs, err := NewCreateImageTool(nil).resolveReferenceImages(ctx, map[string]any{
|
||||
"ref_images": []any{map[string]any{"path": "inputs/reference.png"}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("resolve staged create_image reference: %v", err)
|
||||
}
|
||||
if len(refs) != 1 || string(refs[0].Data) != string(want) {
|
||||
t.Fatalf("resolved refs = %#v, want staged reference", refs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelegationArtifactResultPolicySuppressesUnpublishedMedia(t *testing.T) {
|
||||
ctx, _, outputs := delegationArtifactToolContext(t)
|
||||
result := &Result{
|
||||
ForLLM: "created\nMEDIA:" + filepath.Join(outputs, "generated.png") + "\nkeep this",
|
||||
Media: []bus.MediaFile{{
|
||||
Path: filepath.Join(outputs, "generated.png"),
|
||||
}},
|
||||
}
|
||||
|
||||
ApplyDelegationArtifactResultPolicy(ctx, result)
|
||||
|
||||
if len(result.Media) != 0 {
|
||||
t.Fatalf("unpublished media remained: %#v", result.Media)
|
||||
}
|
||||
if strings.Contains(result.ForLLM, "MEDIA:") || !strings.Contains(result.ForLLM, "keep this") {
|
||||
t.Fatalf("artifact result policy = %q", result.ForLLM)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelegationArtifactResultPolicyPreservesMediaDiscussion(t *testing.T) {
|
||||
ctx := delegationArtifactTestContext()
|
||||
result := &Result{
|
||||
ForLLM: "The literal marker MEDIA: is documented here.\nGenerated MEDIA:outputs/report.pdf successfully.",
|
||||
}
|
||||
|
||||
ApplyDelegationArtifactResultPolicy(ctx, result)
|
||||
|
||||
if !strings.Contains(result.ForLLM, "The literal marker MEDIA: is documented here.") {
|
||||
t.Fatalf("legitimate MEDIA discussion was removed: %q", result.ForLLM)
|
||||
}
|
||||
if strings.Contains(result.ForLLM, "MEDIA:outputs/report.pdf") {
|
||||
t.Fatalf("unpublished media path leaked: %q", result.ForLLM)
|
||||
}
|
||||
if !strings.Contains(result.ForLLM, "Generated successfully.") {
|
||||
t.Fatalf("surrounding tool result text was removed: %q", result.ForLLM)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ApplyDelegationArtifactResultPolicy keeps unvalidated delegate outputs inside
|
||||
// the exchange. Publication is owned by DelegateTool after the child run
|
||||
// succeeds and the manifest has been validated.
|
||||
func ApplyDelegationArtifactResultPolicy(ctx context.Context, result *Result) {
|
||||
if result == nil || !IsDelegationArtifactRun(ctx) {
|
||||
return
|
||||
}
|
||||
result.Media = nil
|
||||
result.ForLLM = stripArtifactMediaLines(result.ForLLM)
|
||||
}
|
||||
|
||||
func stripArtifactMediaLines(content string) string {
|
||||
if !strings.Contains(content, "MEDIA:") {
|
||||
return content
|
||||
}
|
||||
lines := strings.Split(content, "\n")
|
||||
kept := make([]string, 0, len(lines))
|
||||
for _, line := range lines {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "[[audio_as_voice]]") {
|
||||
continue
|
||||
}
|
||||
cleaned := strings.TrimRight(embeddedMediaPattern.ReplaceAllString(line, ""), " \t")
|
||||
if strings.TrimSpace(cleaned) != "" {
|
||||
kept = append(kept, cleaned)
|
||||
}
|
||||
}
|
||||
return strings.TrimSpace(strings.Join(kept, "\n"))
|
||||
}
|
||||
Loaded 100 of 186 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user