Files
goclaw/internal/tools/delegate_tool_hooks_test.go

2349 lines
71 KiB
Go

package tools
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/google/uuid"
"github.com/nextlevelbuilder/goclaw/internal/bus"
orchestration "github.com/nextlevelbuilder/goclaw/internal/childrun"
"github.com/nextlevelbuilder/goclaw/internal/config"
"github.com/nextlevelbuilder/goclaw/internal/eventbus"
"github.com/nextlevelbuilder/goclaw/internal/hooks"
"github.com/nextlevelbuilder/goclaw/internal/store"
"github.com/nextlevelbuilder/goclaw/internal/tracing"
)
// fakeDispatcher records Fire calls and returns a preset decision.
type fakeDispatcher struct {
decision hooks.Decision
calls int
}
type recordingDelegateEventBus struct {
mu sync.Mutex
events []eventbus.DomainEvent
}
func (b *recordingDelegateEventBus) Publish(event eventbus.DomainEvent) {
b.mu.Lock()
defer b.mu.Unlock()
b.events = append(b.events, event)
}
func (*recordingDelegateEventBus) Subscribe(
eventbus.EventType,
eventbus.DomainEventHandler,
) func() {
return func() {}
}
func (*recordingDelegateEventBus) Start(context.Context) {}
func (*recordingDelegateEventBus) Drain(time.Duration) error { return nil }
func (b *recordingDelegateEventBus) snapshot() []eventbus.DomainEvent {
b.mu.Lock()
defer b.mu.Unlock()
return append([]eventbus.DomainEvent(nil), b.events...)
}
func (f *fakeDispatcher) Fire(_ context.Context, _ hooks.Event) (hooks.FireResult, error) {
f.calls++
return hooks.FireResult{Decision: f.decision}, nil
}
// --- minimal noop stores to satisfy AgentLinkStore / AgentCRUDStore ---
type noopAgentLink struct{}
var noopAgentLinkID = uuid.MustParse("8d9c2f78-7914-4e4e-b75f-e6e22ea75921")
func (noopAgentLink) CreateLink(_ context.Context, _ *store.AgentLinkData) error { return nil }
func (noopAgentLink) DeleteLink(_ context.Context, _ uuid.UUID) error { return nil }
func (noopAgentLink) UpdateLink(_ context.Context, _ uuid.UUID, _ map[string]any) error {
return nil
}
func (noopAgentLink) GetLink(_ context.Context, _ uuid.UUID) (*store.AgentLinkData, error) {
return nil, nil
}
func (noopAgentLink) ListLinksFrom(_ context.Context, _ uuid.UUID) ([]store.AgentLinkData, error) {
return nil, nil
}
func (noopAgentLink) ListLinksTo(_ context.Context, _ uuid.UUID) ([]store.AgentLinkData, error) {
return nil, nil
}
func (noopAgentLink) CanDelegate(_ context.Context, _, _ uuid.UUID) (bool, error) {
return true, nil
}
func (noopAgentLink) GetLinkBetween(_ context.Context, _, _ uuid.UUID) (*store.AgentLinkData, error) {
return &store.AgentLinkData{
BaseModel: store.BaseModel{ID: noopAgentLinkID},
MaxConcurrent: 3,
Status: store.LinkStatusActive,
}, nil
}
func (noopAgentLink) DelegateTargets(_ context.Context, _ uuid.UUID) ([]store.AgentLinkData, error) {
return nil, nil
}
func (noopAgentLink) SearchDelegateTargets(_ context.Context, _ uuid.UUID, _ string, _ int) ([]store.AgentLinkData, error) {
return nil, nil
}
func (noopAgentLink) SearchDelegateTargetsByEmbedding(_ context.Context, _ uuid.UUID, _ []float32, _ int) ([]store.AgentLinkData, error) {
return nil, nil
}
func (noopAgentLink) DeleteTeamLinksForAgent(_ context.Context, _, _ uuid.UUID) error { return nil }
type fixedAgentLink struct {
noopAgentLink
link store.AgentLinkData
}
func (f fixedAgentLink) GetLinkBetween(_ context.Context, _, _ uuid.UUID) (*store.AgentLinkData, error) {
link := f.link
return &link, nil
}
type noopAgentCRUD struct {
keyToID map[string]uuid.UUID
}
func (n noopAgentCRUD) Create(_ context.Context, _ *store.AgentData) error { return nil }
func (n noopAgentCRUD) GetByKey(_ context.Context, key string) (*store.AgentData, error) {
id, ok := n.keyToID[key]
if !ok {
id = uuid.New()
}
return &store.AgentData{BaseModel: store.BaseModel{ID: id}, AgentKey: key}, nil
}
func (n noopAgentCRUD) GetByID(_ context.Context, _ uuid.UUID) (*store.AgentData, error) {
return nil, nil
}
func (n noopAgentCRUD) GetByIDUnscoped(_ context.Context, _ uuid.UUID) (*store.AgentData, error) {
return nil, nil
}
func (n noopAgentCRUD) GetByKeys(_ context.Context, _ []string) ([]store.AgentData, error) {
return nil, nil
}
func (n noopAgentCRUD) GetByIDs(_ context.Context, _ []uuid.UUID) ([]store.AgentData, error) {
return nil, nil
}
func (n noopAgentCRUD) Update(_ context.Context, _ uuid.UUID, _ map[string]any) error { return nil }
func (n noopAgentCRUD) Delete(_ context.Context, _ uuid.UUID) error { return nil }
func (n noopAgentCRUD) List(_ context.Context, _ string) ([]store.AgentData, error) { return nil, nil }
func (n noopAgentCRUD) GetDefault(_ context.Context) (*store.AgentData, error) { return nil, nil }
func (n noopAgentCRUD) ResetStuckSummoning(_ context.Context) (int64, error) { return 0, nil }
// --- helpers ---
func makeDelegateCtx(t *testing.T) context.Context {
t.Helper()
ctx := store.WithAgentID(context.Background(), uuid.New())
ctx = store.WithTenantID(ctx, store.MasterTenantID)
ctx = store.WithAgentKey(ctx, "parent-agent")
ctx = WithToolWorkspace(ctx, t.TempDir())
return ctx
}
func newDelegateTestTool(
t *testing.T,
links store.AgentLinkStore,
runFn DelegateRunFunc,
) *DelegateTool {
t.Helper()
tool := NewDelegateTool(links, noopAgentCRUD{}, nil, runFn)
tool.SetWorkspace(t.TempDir())
t.Cleanup(tool.Close)
return tool
}
// --- tests ---
func TestDelegateTool_EmitsSentOnlyAfterAdmissionWithLogicalTask(t *testing.T) {
t.Run("rejected admission emits nothing", func(t *testing.T) {
admission := orchestration.NewChildRunAdmission(1, 1)
if err := admission.Close(context.Background()); err != nil {
t.Fatal(err)
}
events := &recordingDelegateEventBus{}
tool := NewDelegateToolWithAdmission(
noopAgentLink{},
noopAgentCRUD{},
events,
func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
t.Fatal("runFn called after admission closed")
return DelegateResult{}, nil
},
admission,
)
tool.SetWorkspace(t.TempDir())
defer tool.Close()
result := tool.Execute(makeDelegateCtx(t), map[string]any{
"agent_key": "child-agent",
"task": "do something",
"mode": "sync",
})
if result == nil || !result.IsError {
t.Fatalf("result = %#v, want admission error", result)
}
if got := events.snapshot(); len(got) != 0 {
t.Fatalf("events = %#v, want none", got)
}
})
t.Run("accepted dispatch redacts caller host path", func(t *testing.T) {
events := &recordingDelegateEventBus{}
managedWorkspace := t.TempDir()
callerWorkspace := t.TempDir()
tool := NewDelegateTool(
noopAgentLink{},
noopAgentCRUD{},
events,
func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
return DelegateResult{Content: "done"}, nil
},
)
tool.SetWorkspace(managedWorkspace)
defer tool.Close()
ctx := WithToolWorkspace(makeDelegateCtx(t), callerWorkspace)
result := tool.Execute(ctx, map[string]any{
"agent_key": "child-agent",
"task": "inspect " + filepath.Join(callerWorkspace, "report.txt"),
"mode": "sync",
})
if result == nil || result.IsError {
t.Fatalf("result = %#v, want success", result)
}
eventsSnapshot := events.snapshot()
if len(eventsSnapshot) == 0 {
t.Fatal("delegate.sent not emitted")
}
var sent *eventbus.DelegateSentPayload
for _, event := range eventsSnapshot {
if event.Type != eventbus.EventDelegateSent {
continue
}
payload, ok := event.Payload.(eventbus.DelegateSentPayload)
if !ok {
t.Fatalf("delegate.sent payload = %T", event.Payload)
}
sent = &payload
break
}
if sent == nil {
t.Fatalf("events = %#v, want delegate.sent", eventsSnapshot)
}
if strings.Contains(sent.Task, callerWorkspace) ||
!strings.Contains(sent.Task, "caller workspace") {
t.Fatalf("delegate.sent task = %q", sent.Task)
}
})
}
func TestDelegateTool_SyncQueuedTimeoutDoesNotReportCompletion(t *testing.T) {
admission := orchestration.NewChildRunAdmission(1, 4)
blockerStarted := make(chan struct{})
blockerRelease := make(chan struct{})
blocker, err := admission.Enqueue(
context.Background(),
orchestration.ChildRunConstraints{TaskID: "blocker"},
func(context.Context, *orchestration.ChildRunLease) {
close(blockerStarted)
<-blockerRelease
},
)
if err != nil {
t.Fatal(err)
}
if err := blocker.Activate(); err != nil {
t.Fatal(err)
}
select {
case <-blockerStarted:
case <-time.After(time.Second):
t.Fatal("blocker did not start")
}
events := &recordingDelegateEventBus{}
runCalled := false
tool := NewDelegateToolWithAdmission(
noopAgentLink{},
noopAgentCRUD{},
events,
func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
runCalled = true
return DelegateResult{Content: "unexpected"}, nil
},
admission,
)
tool.SetWorkspace(t.TempDir())
defer tool.Close()
ctx, cancel := context.WithTimeout(makeDelegateCtx(t), 20*time.Millisecond)
defer cancel()
result := tool.Execute(ctx, map[string]any{
"agent_key": "child-agent",
"task": "must time out while queued",
"mode": "sync",
})
if result == nil || !result.IsError {
t.Fatalf("result = %#v, want queued timeout", result)
}
if runCalled {
t.Fatal("queued timed-out delegation ran")
}
for _, event := range events.snapshot() {
if event.Type == eventbus.EventDelegateSent ||
event.Type == eventbus.EventDelegateCompleted {
t.Fatalf("timed-out queued delegation emitted %q", event.Type)
}
}
close(blockerRelease)
<-blocker.Done()
if err := admission.Close(context.Background()); err != nil {
t.Fatal(err)
}
}
func TestDelegateTool_SyncCallbackPanicReportsFailure(t *testing.T) {
events := &recordingDelegateEventBus{}
tool := NewDelegateTool(
noopAgentLink{},
noopAgentCRUD{},
events,
func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
panic("injected delegate panic")
},
)
tool.SetWorkspace(t.TempDir())
defer tool.Close()
result := tool.Execute(makeDelegateCtx(t), map[string]any{
"agent_key": "child-agent",
"task": "panic safely",
"mode": "sync",
})
if result == nil || !result.IsError ||
!strings.Contains(result.ForLLM, "child run panic") {
t.Fatalf("result = %#v, want recovered callback panic", result)
}
var failed, completed bool
for _, event := range events.snapshot() {
failed = failed || event.Type == eventbus.EventDelegateFailed
completed = completed || event.Type == eventbus.EventDelegateCompleted
}
if !failed || completed {
t.Fatalf("events = %#v, want failed without completed", events.snapshot())
}
}
func TestDelegateTool_SubagentStartBlock_AbortsDispatch(t *testing.T) {
runCalled := 0
runFn := func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
runCalled++
return DelegateResult{Content: "ok"}, nil
}
tool := newDelegateTestTool(t, noopAgentLink{}, runFn)
disp := &fakeDispatcher{decision: hooks.DecisionBlock}
tool.SetHookDispatcher(disp)
result := tool.Execute(makeDelegateCtx(t), map[string]any{
"agent_key": "child-agent",
"task": "do something",
"mode": "sync",
})
if result == nil || !result.IsError {
t.Fatal("expected error result when hook blocks")
}
if runCalled != 0 {
t.Errorf("runFn must not be called; got %d calls", runCalled)
}
if disp.calls != 1 {
t.Errorf("expected 1 dispatcher Fire call; got %d", disp.calls)
}
}
func TestDelegateTool_SubagentStartAllow_ProceedsToRun(t *testing.T) {
runCalled := 0
runFn := func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
runCalled++
return DelegateResult{Content: "done"}, nil
}
tool := newDelegateTestTool(t, noopAgentLink{}, runFn)
disp := &fakeDispatcher{decision: hooks.DecisionAllow}
tool.SetHookDispatcher(disp)
result := tool.Execute(makeDelegateCtx(t), map[string]any{
"agent_key": "child-agent",
"task": "do something",
"mode": "sync",
})
if result != nil && result.IsError {
t.Fatalf("unexpected error: %s", result.ForLLM)
}
if runCalled != 1 {
t.Errorf("expected runFn called once; got %d", runCalled)
}
}
func TestDelegateTool_NilDispatcher_SkipsHook(t *testing.T) {
runCalled := 0
runFn := func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
runCalled++
return DelegateResult{Content: "done"}, nil
}
// No SetHookDispatcher — hookDispatcher stays nil.
tool := newDelegateTestTool(t, noopAgentLink{}, runFn)
result := tool.Execute(makeDelegateCtx(t), map[string]any{
"agent_key": "child-agent",
"task": "do something",
"mode": "sync",
})
if result != nil && result.IsError {
t.Fatalf("unexpected error: %s", result.ForLLM)
}
if runCalled != 1 {
t.Errorf("expected runFn called once; got %d", runCalled)
}
}
func TestDelegateTool_SyncNestedRunCompletesBeforeOuterPublication(t *testing.T) {
admission := orchestration.NewChildRunAdmission(1, 8)
t.Cleanup(func() {
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if err := admission.Close(ctx); err != nil {
t.Errorf("close admission: %v", err)
}
})
callerWorkspace := t.TempDir()
managedWorkspace := t.TempDir()
childB := uuid.New()
childCStarted := make(chan struct{})
releaseChildC := make(chan struct{})
var childCCompleted atomic.Bool
var tool *DelegateTool
runFn := func(ctx context.Context, req DelegateRequest) (DelegateResult, error) {
switch req.ToAgentKey {
case "child-b":
nestedCtx := store.WithAgentID(ctx, childB)
nestedCtx = store.WithAgentKey(nestedCtx, "child-b")
nestedCtx = WithToolAgentKey(nestedCtx, "child-b")
nestedCtx = WithToolWorkspace(nestedCtx, req.DelegateOutputsPath)
nestedCtx = WithDelegationID(nestedCtx, req.DelegationID)
nestedCtx = WithDelegationArtifactInputs(nestedCtx, req.DelegateInputsPath)
nestedCtx = WithSubagentConfig(nestedCtx, &config.SubagentsConfig{MaxSpawnDepth: 1})
if depth := subagentDepthFromContext(nestedCtx, -1); depth != 0 {
return DelegateResult{}, fmt.Errorf("delegated agent depth = %d, want 0", depth)
}
if parentTaskID := subagentTaskIDFromContext(nestedCtx); parentTaskID != "" {
return DelegateResult{}, fmt.Errorf("delegated agent parent task = %q, want empty", parentTaskID)
}
if scope := subagentScopeFromContext(nestedCtx); scope.RootAgentID != childB ||
scope.RootAgentKey != "child-b" {
return DelegateResult{}, fmt.Errorf("delegated agent scope = %#v, want child-b", scope)
}
nested := tool.Execute(nestedCtx, map[string]any{
"agent_key": "child-c",
"task": "finish before child B returns",
"mode": "sync",
})
if nested == nil || nested.IsError {
return DelegateResult{}, fmt.Errorf("nested delegation failed: %#v", nested)
}
if !childCCompleted.Load() {
return DelegateResult{}, errors.New("nested delegation returned before child C completed")
}
if err := os.WriteFile(
filepath.Join(req.DelegateOutputsPath, "b-after-c.txt"),
[]byte("child C completed"),
0600,
); err != nil {
return DelegateResult{}, err
}
return DelegateResult{Content: "child B completed after child C"}, nil
case "child-c":
close(childCStarted)
select {
case <-releaseChildC:
case <-ctx.Done():
return DelegateResult{}, ctx.Err()
}
childCCompleted.Store(true)
return DelegateResult{Content: "child C completed"}, nil
default:
return DelegateResult{}, fmt.Errorf("unexpected target %q", req.ToAgentKey)
}
}
tool = NewDelegateToolWithAdmission(
noopAgentLink{},
noopAgentCRUD{keyToID: map[string]uuid.UUID{
"child-b": childB,
"child-c": uuid.New(),
}},
nil,
runFn,
admission,
)
tool.SetWorkspace(managedWorkspace)
t.Cleanup(tool.Close)
ctx := WithToolWorkspace(makeDelegateCtx(t), callerWorkspace)
ctx = WithSubagentConfig(ctx, &config.SubagentsConfig{MaxSpawnDepth: 1})
resultCh := make(chan *Result, 1)
go func() {
resultCh <- tool.Execute(ctx, map[string]any{
"agent_key": "child-b",
"task": "delegate synchronously to child C",
"mode": "sync",
})
}()
select {
case <-childCStarted:
case <-time.After(time.Second):
t.Fatal("nested child C did not start")
}
select {
case result := <-resultCh:
t.Fatalf("outer delegation published before nested child completed: %#v", result)
case <-time.After(50 * time.Millisecond):
}
if entries, err := os.ReadDir(filepath.Join(callerWorkspace, ".delegations")); err == nil && len(entries) > 0 {
t.Fatalf("outer artifacts visible while nested child was active: %#v", entries)
} else if err != nil && !errors.Is(err, os.ErrNotExist) {
t.Fatalf("inspect caller publication root: %v", err)
}
close(releaseChildC)
var result *Result
select {
case result = <-resultCh:
case <-time.After(2 * time.Second):
t.Fatal("outer delegation did not finish after nested child completed")
}
if result == nil || result.IsError {
t.Fatalf("outer delegation = %#v, want success", result)
}
foundMarker := false
for _, media := range result.Media {
if media.Filename == "b-after-c.txt" {
foundMarker = true
content, err := os.ReadFile(media.Path)
if err != nil {
t.Fatalf("read published marker: %v", err)
}
if string(content) != "child C completed" {
t.Fatalf("published marker = %q", content)
}
}
}
if !foundMarker {
t.Fatalf("published media = %#v, want b-after-c.txt", result.Media)
}
}
func TestDelegateTool_ConcurrentMultiLinkTopologiesPublishInCallerScope(t *testing.T) {
managedWorkspace := t.TempDir()
callerA := t.TempDir()
callerC := t.TempDir()
agentA := uuid.New()
agentB := uuid.New()
agentC := uuid.New()
started := make(chan struct{}, 3)
release := make(chan struct{})
runFn := func(ctx context.Context, req DelegateRequest) (DelegateResult, error) {
started <- struct{}{}
select {
case <-release:
case <-ctx.Done():
return DelegateResult{}, ctx.Err()
}
if err := os.WriteFile(
filepath.Join(req.DelegateOutputsPath, "result.txt"),
[]byte(req.Task),
0600,
); err != nil {
return DelegateResult{}, err
}
return DelegateResult{Content: req.Task}, nil
}
tool := NewDelegateTool(
noopAgentLink{},
noopAgentCRUD{keyToID: map[string]uuid.UUID{
"agent-b": agentB,
"agent-c": agentC,
}},
nil,
runFn,
)
tool.SetWorkspace(managedWorkspace)
t.Cleanup(tool.Close)
type topologyCase struct {
name string
fromID uuid.UUID
fromKey string
toKey string
callerWorkspace string
payload string
}
cases := []topologyCase{
{
name: "a-to-b",
fromID: agentA,
fromKey: "agent-a",
toKey: "agent-b",
callerWorkspace: callerA,
payload: "A to B",
},
{
name: "a-to-c",
fromID: agentA,
fromKey: "agent-a",
toKey: "agent-c",
callerWorkspace: callerA,
payload: "A to C",
},
{
name: "c-to-b",
fromID: agentC,
fromKey: "agent-c",
toKey: "agent-b",
callerWorkspace: callerC,
payload: "C to B",
},
}
results := make([]*Result, len(cases))
var wg sync.WaitGroup
for i := range cases {
wg.Add(1)
go func(i int) {
defer wg.Done()
tc := cases[i]
ctx := store.WithAgentID(context.Background(), tc.fromID)
ctx = store.WithAgentKey(ctx, tc.fromKey)
ctx = store.WithTenantID(ctx, store.MasterTenantID)
ctx = WithToolWorkspace(ctx, tc.callerWorkspace)
results[i] = tool.Execute(ctx, map[string]any{
"agent_key": tc.toKey,
"task": tc.payload,
"mode": "sync",
})
}(i)
}
for range cases {
select {
case <-started:
case <-time.After(time.Second):
t.Fatal("concurrent delegation did not start")
}
}
close(release)
wg.Wait()
aPublications := make(map[string]struct{})
for i, tc := range cases {
result := results[i]
if result == nil || result.IsError {
t.Fatalf("%s result = %#v", tc.name, result)
}
if len(result.Media) != 1 {
t.Fatalf("%s media = %#v, want one output", tc.name, result.Media)
}
media := result.Media[0]
relative, err := filepath.Rel(tc.callerWorkspace, media.Path)
if err != nil || strings.HasPrefix(relative, "..") {
t.Fatalf("%s output %q escaped caller %q", tc.name, media.Path, tc.callerWorkspace)
}
parts := strings.Split(filepath.ToSlash(relative), "/")
if len(parts) != 4 || parts[0] != ".delegations" || parts[2] != "outputs" || parts[3] != "result.txt" {
t.Fatalf("%s relative output = %q", tc.name, relative)
}
content, err := os.ReadFile(media.Path)
if err != nil {
t.Fatalf("%s read output: %v", tc.name, err)
}
if string(content) != tc.payload {
t.Fatalf("%s output = %q, want %q", tc.name, content, tc.payload)
}
if tc.fromKey == "agent-a" {
if _, exists := aPublications[parts[1]]; exists {
t.Fatalf("A→B and A→C reused delegation %q", parts[1])
}
aPublications[parts[1]] = struct{}{}
}
}
if len(aPublications) != 2 {
t.Fatalf("A publications = %d, want 2 isolated delegation roots", len(aPublications))
}
}
func TestDelegateTool_LinkMaxConcurrentIsReservedAndUnenforced(t *testing.T) {
linkID := uuid.New()
started := make(chan struct{})
releaseRun := make(chan struct{})
var calls atomic.Int32
runFn := func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
call := calls.Add(1)
if call == 1 {
close(started)
}
if call <= 2 {
<-releaseRun
}
return DelegateResult{Content: "done"}, nil
}
tool := newDelegateTestTool(t, fixedAgentLink{link: store.AgentLinkData{
BaseModel: store.BaseModel{ID: linkID},
MaxConcurrent: 1,
Status: store.LinkStatusActive,
}}, runFn)
first := tool.Execute(makeDelegateCtx(t), map[string]any{
"agent_key": "child-agent",
"task": "first",
"mode": "async",
})
if first == nil || first.IsError {
t.Fatalf("first delegation = %#v, want accepted", first)
}
select {
case <-started:
case <-time.After(time.Second):
t.Fatal("first async delegation did not start")
}
second := tool.Execute(makeDelegateCtx(t), map[string]any{
"agent_key": "child-agent",
"task": "second",
"mode": "async",
})
if second == nil || second.IsError {
t.Fatalf("second delegation = %#v, want shared admission to accept it", second)
}
deadline := time.Now().Add(time.Second)
for calls.Load() < 2 && time.Now().Before(deadline) {
time.Sleep(time.Millisecond)
}
if got := calls.Load(); got != 2 {
t.Fatalf("runFn calls = %d, want max_concurrent ignored", got)
}
close(releaseRun)
third := tool.Execute(makeDelegateCtx(t), map[string]any{
"agent_key": "child-agent",
"task": "third",
"mode": "sync",
})
if third == nil || third.IsError {
t.Fatalf("third delegation = %#v, want success after release", third)
}
}
func TestDelegateTool_LinkSlotReleasedOnHookBlockAndSyncError(t *testing.T) {
link := fixedAgentLink{link: store.AgentLinkData{
BaseModel: store.BaseModel{ID: uuid.New()},
MaxConcurrent: 1,
Status: store.LinkStatusActive,
}}
var calls atomic.Int32
runFn := func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
if calls.Add(1) == 1 {
return DelegateResult{}, errors.New("first run failed")
}
return DelegateResult{Content: "done"}, nil
}
tool := newDelegateTestTool(t, link, runFn)
disp := &fakeDispatcher{decision: hooks.DecisionBlock}
tool.SetHookDispatcher(disp)
blocked := tool.Execute(makeDelegateCtx(t), map[string]any{
"agent_key": "child-agent",
"task": "blocked",
"mode": "sync",
})
if blocked == nil || !blocked.IsError {
t.Fatalf("blocked delegation = %#v, want error", blocked)
}
disp.decision = hooks.DecisionAllow
failed := tool.Execute(makeDelegateCtx(t), map[string]any{
"agent_key": "child-agent",
"task": "fails",
"mode": "sync",
})
if failed == nil || !failed.IsError ||
strings.Contains(failed.ForLLM, "active delegation limit reached") {
t.Fatalf("failed delegation = %#v, want run error after hook slot release", failed)
}
succeeded := tool.Execute(makeDelegateCtx(t), map[string]any{
"agent_key": "child-agent",
"task": "succeeds",
"mode": "sync",
})
if succeeded == nil || succeeded.IsError {
t.Fatalf("delegation after sync error = %#v, want success", succeeded)
}
}
func TestDelegateTool_PreservesGroupAuthorizationScope(t *testing.T) {
var captured DelegateRequest
runFn := func(_ context.Context, req DelegateRequest) (DelegateResult, error) {
captured = req
return DelegateResult{Content: "done"}, nil
}
tool := newDelegateTestTool(t, noopAgentLink{}, runFn)
ctx := makeDelegateCtx(t)
ctx = store.WithUserID(ctx, "group:telegram:-100123")
ctx = store.WithSenderID(ctx, "386246614")
ctx = store.WithRole(ctx, "viewer")
ctx = WithToolChannel(ctx, "telegram-main")
ctx = WithToolChannelType(ctx, "telegram")
ctx = WithToolChatID(ctx, "-100123")
ctx = WithToolPeerKind(ctx, "group")
result := tool.Execute(ctx, map[string]any{
"agent_key": "child-agent",
"task": "create a file",
"mode": "sync",
})
if result == nil || result.IsError {
t.Fatalf("delegate result = %#v, want success", result)
}
if captured.UserID != "group:telegram:-100123" {
t.Fatalf("UserID = %q, want group authorization scope", captured.UserID)
}
if captured.SenderID != "386246614" || captured.Role != "viewer" {
t.Fatalf("actor scope = (%q, %q), want real sender and role", captured.SenderID, captured.Role)
}
if captured.Channel != "telegram-main" || captured.ChannelType != "telegram" ||
captured.ChatID != "-100123" || captured.PeerKind != "group" {
t.Fatalf("origin scope = %#v", captured)
}
}
func TestDelegateTool_CurrentRunMediaBecomesReadOnlyRelativeInput(t *testing.T) {
workspace := t.TempDir()
source := filepath.Join(workspace, "reference.jpg")
if err := os.WriteFile(source, []byte("reference"), 0644); err != nil {
t.Fatal(err)
}
var captured DelegateRequest
runFn := func(_ context.Context, req DelegateRequest) (DelegateResult, error) {
captured = req
staged, err := os.ReadFile(filepath.Join(req.DelegateInputsPath, "reference.jpg"))
if err != nil {
return DelegateResult{}, err
}
if string(staged) != "reference" {
return DelegateResult{}, errors.New("staged input content mismatch")
}
return DelegateResult{Content: "done"}, nil
}
tool := newDelegateTestTool(t, noopAgentLink{}, runFn)
ctx := WithToolWorkspace(makeDelegateCtx(t), workspace)
ctx = WithRunMediaPaths(ctx, []string{source})
result := tool.Execute(ctx, map[string]any{
"agent_key": "child-agent",
"task": "use " + source,
"mode": "sync",
})
if result == nil || result.IsError {
t.Fatalf("delegate result = %#v, want success", result)
}
if strings.Contains(captured.Task, source) || !strings.Contains(captured.Task, "inputs/reference.jpg") {
t.Fatalf("Task = %q, want only logical input alias", captured.Task)
}
if captured.DelegateInputsPath == "" || captured.DelegateOutputsPath == "" {
t.Fatalf("runtime artifact roots missing: %#v", captured)
}
}
func TestDelegateTool_DoesNotScrapeAbsolutePathsFromTask(t *testing.T) {
workspace := t.TempDir()
source := filepath.Join(workspace, "family-reference.jpg")
if err := os.WriteFile(source, []byte("reference"), 0644); err != nil {
t.Fatal(err)
}
var captured DelegateRequest
var stagedInputCount int
runFn := func(_ context.Context, req DelegateRequest) (DelegateResult, error) {
captured = req
entries, err := os.ReadDir(req.DelegateInputsPath)
if err != nil {
return DelegateResult{}, err
}
stagedInputCount = len(entries)
return DelegateResult{Content: "done"}, nil
}
tool := newDelegateTestTool(t, noopAgentLink{}, runFn)
ctx := WithToolWorkspace(makeDelegateCtx(t), workspace)
result := tool.Execute(ctx, map[string]any{
"agent_key": "child-agent",
"task": "Retry using the exact sample file path: " + source + ".",
"mode": "sync",
})
if result == nil || result.IsError {
t.Fatalf("delegate result = %#v, want success", result)
}
if stagedInputCount != 0 {
t.Fatalf("staged input count = %d, want no free-form path scraping", stagedInputCount)
}
if !strings.Contains(captured.Task, source) {
t.Fatalf("Task = %q, want untrusted prose left as prose", captured.Task)
}
}
func TestDelegateTool_RejectsInvalidExplicitInputBeforeDispatch(t *testing.T) {
workspace := t.TempDir()
var called atomic.Bool
runFn := func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
called.Store(true)
return DelegateResult{Content: "done"}, nil
}
tool := newDelegateTestTool(t, noopAgentLink{}, runFn)
ctx := WithToolWorkspace(makeDelegateCtx(t), workspace)
result := tool.Execute(ctx, map[string]any{
"agent_key": "child-agent",
"task": "Use the requested input",
"mode": "sync",
"inputs": []any{"../outside-reference.jpg"},
})
if result == nil || !result.IsError {
t.Fatalf("delegate result = %#v, want validation error", result)
}
if called.Load() {
t.Fatal("runFn called for invalid explicit input")
}
}
func TestDelegateTool_MissingManagedWorkspaceFailsClosed(t *testing.T) {
tool := NewDelegateTool(noopAgentLink{}, noopAgentCRUD{}, nil, func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
return DelegateResult{Content: "unexpected"}, nil
})
result := tool.Execute(makeDelegateCtx(t), map[string]any{
"agent_key": "child-agent",
"task": "do something",
"mode": "sync",
})
if result == nil || !result.IsError {
t.Fatalf("delegate result = %#v, want fail-closed setup error", result)
}
}
func TestDelegateTool_PublishesOnlyManifestOutputsAndCleansExchange(t *testing.T) {
workspace := t.TempDir()
managedWorkspace := t.TempDir()
rawMedia := filepath.Join(t.TempDir(), "raw-child-media.png")
if err := os.WriteFile(rawMedia, []byte("raw"), 0644); err != nil {
t.Fatal(err)
}
var captured DelegateRequest
runFn := func(_ context.Context, req DelegateRequest) (DelegateResult, error) {
captured = req
if err := os.WriteFile(filepath.Join(req.DelegateOutputsPath, "report.txt"), []byte("published"), 0600); err != nil {
return DelegateResult{}, err
}
return DelegateResult{
Content: "done",
Media: []bus.MediaFile{{
Path: rawMedia,
MimeType: "image/png",
Filename: "raw-child-media.png",
}},
}, nil
}
tool := NewDelegateTool(noopAgentLink{}, noopAgentCRUD{}, nil, runFn)
tool.SetWorkspace(managedWorkspace)
ctx := WithToolWorkspace(makeDelegateCtx(t), workspace)
result := tool.Execute(ctx, map[string]any{
"agent_key": "child-agent",
"task": "create report.txt",
"mode": "sync",
})
if result == nil || result.IsError {
t.Fatalf("delegate result = %#v, want success", result)
}
if len(result.Media) != 1 {
t.Fatalf("result media = %#v, want one published manifest output", result.Media)
}
got := result.Media[0]
if got.Path == rawMedia || got.Filename != "report.txt" || got.MimeType != "text/plain; charset=utf-8" {
t.Fatalf("published media = %#v", got)
}
if content, err := os.ReadFile(got.Path); err != nil || string(content) != "published" {
t.Fatalf("published content = %q, %v", content, err)
}
if _, err := os.Stat(filepath.Dir(captured.DelegateOutputsPath)); !os.IsNotExist(err) {
t.Fatalf("successful exchange retained at runtime path: %v", err)
}
}
func TestDelegateTool_AdmissionConstraintsAreTenantScopedAndRootless(t *testing.T) {
tenantID := uuid.New()
ctx := store.WithTenantID(context.Background(), tenantID)
ctx = withSubagentExecution(ctx, TaskScope{
TenantID: tenantID, RootAgentID: uuid.New(), RootAgentKey: "parent-agent",
}, "parent-task", 3, nil)
req := DelegateRequest{
TenantID: tenantID.String(),
DelegationID: uuid.NewString(),
}
got := delegateAdmissionConstraints(ctx, req)
if got.TenantID != tenantID || got.RootAgentID != uuid.Nil {
t.Fatalf("identity constraints = %#v", got)
}
if got.TaskID != req.DelegationID || got.ParentTaskID != "parent-task" {
t.Fatalf("task constraints = %#v", got)
}
if got.Depth != 4 || got.MaxDepth != 0 {
t.Fatalf("depth constraints = %#v", got)
}
}
func TestDelegateTool_AdmissionDoesNotUseAgentSpawnDepth(t *testing.T) {
req := DelegateRequest{
TenantID: store.MasterTenantID.String(),
DelegationID: uuid.NewString(),
}
got := delegateAdmissionConstraints(context.Background(), req)
if got.Depth != 1 || got.MaxDepth != 0 {
t.Fatalf("top-level admission depth = %#v, want structural depth 1 without agent cap", got)
}
ctx := WithSubagentConfig(context.Background(), &config.SubagentsConfig{MaxSpawnDepth: 4})
got = delegateAdmissionConstraints(ctx, req)
if got.MaxDepth != 0 {
t.Fatalf("configured agent spawn depth leaked into admission = %#v", got)
}
}
func TestDelegateTool_StagesArtifactsOnlyAfterAdmission(t *testing.T) {
workspace := t.TempDir()
managedWorkspace := t.TempDir()
if err := os.WriteFile(filepath.Join(workspace, "input.txt"), []byte("input"), 0600); err != nil {
t.Fatal(err)
}
admission := orchestration.NewChildRunAdmission(1, 4)
defer func() {
if err := admission.Close(context.Background()); err != nil {
t.Errorf("close admission: %v", err)
}
}()
firstStarted := make(chan struct{})
releaseFirst := make(chan struct{})
secondStarted := make(chan struct{})
runFn := func(_ context.Context, req DelegateRequest) (DelegateResult, error) {
if strings.HasPrefix(req.Task, "first") {
close(firstStarted)
<-releaseFirst
} else {
close(secondStarted)
}
return DelegateResult{Content: "done"}, nil
}
tool := NewDelegateToolWithAdmission(noopAgentLink{}, noopAgentCRUD{}, nil, runFn, admission)
tool.SetWorkspace(managedWorkspace)
ctx := WithToolWorkspace(makeDelegateCtx(t), workspace)
first := tool.Execute(ctx, map[string]any{
"agent_key": "child-agent",
"task": "first",
"mode": "async",
})
if first == nil || first.IsError {
t.Fatalf("first delegation = %#v", first)
}
select {
case <-firstStarted:
case <-time.After(time.Second):
t.Fatal("first delegation did not start")
}
second := tool.Execute(ctx, map[string]any{
"agent_key": "child-agent",
"task": "second",
"mode": "async",
"inputs": []any{"input.txt"},
})
if second == nil || second.IsError {
t.Fatalf("second delegation = %#v", second)
}
var accepted struct {
DelegationID string `json:"delegation_id"`
}
if err := json.Unmarshal([]byte(second.ForLLM), &accepted); err != nil {
t.Fatal(err)
}
secondExchange := filepath.Join(
managedWorkspace,
"collaboration",
"delegations",
accepted.DelegationID,
)
if _, err := os.Stat(secondExchange); !os.IsNotExist(err) {
t.Fatalf("pending delegation staged before admission: %v", err)
}
close(releaseFirst)
select {
case <-secondStarted:
case <-time.After(time.Second):
t.Fatal("second delegation did not start after admission")
}
}
func TestDelegateTool_AsyncReleasesAdmissionBeforeAnnouncement(t *testing.T) {
admission := orchestration.NewChildRunAdmission(1, 4)
messageBus := bus.New()
for range 1000 {
messageBus.PublishInbound(bus.InboundMessage{Content: "fill"})
}
firstStarted := make(chan struct{})
secondStarted := make(chan struct{})
var calls atomic.Int32
runFn := func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
switch calls.Add(1) {
case 1:
close(firstStarted)
case 2:
close(secondStarted)
}
return DelegateResult{Content: "done"}, nil
}
tool := NewDelegateToolWithAdmission(noopAgentLink{}, noopAgentCRUD{}, nil, runFn, admission)
tool.SetWorkspace(t.TempDir())
tool.SetMsgBus(messageBus)
taskStore := newRecordingSubagentTaskStore()
tool.SetTaskStore(taskStore)
if tool.announceToParent(DelegateRequest{
DelegationID: "backpressure-probe",
ChatID: "chat-1",
}, "must not block", nil) {
t.Fatal("full message bus accepted an announcement")
}
ctx := WithToolChatID(makeDelegateCtx(t), "chat-1")
ctx = WithToolSessionKey(ctx, "delegate-parent-session")
first := tool.Execute(ctx, map[string]any{
"agent_key": "child-agent",
"task": "first",
"mode": "async",
})
if first == nil || first.IsError {
t.Fatalf("first delegation = %#v, want accepted", first)
}
select {
case <-firstStarted:
case <-time.After(time.Second):
t.Fatal("first delegation did not start")
}
second := tool.Execute(ctx, map[string]any{
"agent_key": "child-agent",
"task": "second",
"mode": "async",
})
if second == nil || second.IsError {
t.Fatalf("second delegation = %#v, want accepted", second)
}
select {
case <-secondStarted:
case <-time.After(time.Second):
t.Fatal("blocked announce retained the only child-run permit")
}
for range 2 {
select {
case metadata := <-taskStore.metadata:
if metadata[asyncCompletionDeliveryKey] != asyncCompletionDeliveryMissed {
t.Fatalf("announcement metadata = %#v, want undelivered", metadata)
}
case <-time.After(time.Second):
t.Fatal("delegation missed announcement was not recorded after bus saturation")
}
}
if err := admission.Close(context.Background()); err != nil {
t.Fatalf("close admission: %v", err)
}
tool.Close()
}
func TestDelegateToolCloseContextDrainsAsyncCompletion(t *testing.T) {
started := make(chan struct{})
release := make(chan struct{})
tool := newDelegateTestTool(t, noopAgentLink{}, func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
close(started)
<-release
return DelegateResult{Content: "done"}, nil
})
tool.SetTaskStore(newRecordingSubagentTaskStore())
result := tool.Execute(makeDelegateCtx(t), map[string]any{
"agent_key": "child-agent",
"task": "wait for shutdown drain",
"mode": "async",
})
if result == nil || result.IsError {
t.Fatalf("delegate result = %#v, want accepted", result)
}
select {
case <-started:
case <-time.After(time.Second):
t.Fatal("delegation did not start")
}
timeoutCtx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
defer cancel()
if err := tool.CloseContext(timeoutCtx); err == nil {
t.Fatal("CloseContext returned before async completion finished")
}
close(release)
if err := tool.CloseContext(context.Background()); err != nil {
t.Fatalf("CloseContext after completion: %v", err)
}
}
func TestDelegationTaskWithInputAliasesDoesNotRewriteProseSubstrings(t *testing.T) {
task := "Use the data file to update the database metadata."
inputs := []delegateInput{{
relativePath: "data",
}}
staged := []DelegationArtifact{{Path: "inputs/data"}}
got := delegationTaskWithInputAliases(task, inputs, staged)
if !strings.HasPrefix(got, task) {
t.Fatalf("task prose was rewritten: %q", got)
}
if !strings.Contains(got, "data => inputs/data") {
t.Fatalf("task is missing the explicit input mapping: %q", got)
}
}
func TestDelegationTaskWithInputAliasesRewritesOnlyExactPathTokens(t *testing.T) {
const reference = "/workspace/agent/.uploads/report.pdf"
task := "Read " + reference + ", but preserve " + reference + ".backup and prefix" + reference
inputs := []delegateInput{{
relativePath: ".uploads/report.pdf",
taskReferences: []string{reference},
}}
staged := []DelegationArtifact{{Path: "inputs/report.pdf"}}
got := delegationTaskWithInputAliases(task, inputs, staged)
if !strings.Contains(got, "Read inputs/report.pdf,") {
t.Fatalf("exact path token was not rewritten: %q", got)
}
if !strings.Contains(got, reference+".backup") {
t.Fatalf("path substring suffix was rewritten: %q", got)
}
if !strings.Contains(got, "prefix"+reference) {
t.Fatalf("path substring prefix was rewritten: %q", got)
}
}
func TestRedactDelegationArtifactTextHandlesSlashVariants(t *testing.T) {
exchange := &DelegationArtifactExchange{
hostRoot: `C:\workspace\collaboration\delegations\123`,
}
text := `failed at C:/workspace/collaboration/delegations/123/inputs/file.txt`
got := redactDelegationArtifactText(text, exchange)
if strings.Contains(got, "C:/workspace") {
t.Fatalf("slash-normalized host path was not redacted: %q", got)
}
if !strings.Contains(got, "inputs/file.txt") {
t.Fatalf("logical input path missing after redaction: %q", got)
}
}
func TestDelegateTool_RetainsFailedExchange(t *testing.T) {
workspace := t.TempDir()
managedWorkspace := t.TempDir()
if err := os.WriteFile(filepath.Join(workspace, "input.txt"), []byte("input"), 0600); err != nil {
t.Fatal(err)
}
var captured DelegateRequest
tool := NewDelegateTool(noopAgentLink{}, noopAgentCRUD{}, nil, func(_ context.Context, req DelegateRequest) (DelegateResult, error) {
captured = req
return DelegateResult{}, errors.New("child run failed")
})
tool.SetWorkspace(managedWorkspace)
t.Cleanup(tool.Close)
ctx := WithToolWorkspace(makeDelegateCtx(t), workspace)
result := tool.Execute(ctx, map[string]any{
"agent_key": "child-agent",
"task": "fail after staging",
"mode": "sync",
"inputs": []any{"input.txt"},
})
if result == nil || !result.IsError {
t.Fatalf("delegate result = %#v, want error", result)
}
exchangeRoot := filepath.Join(
managedWorkspace,
"collaboration",
"delegations",
captured.DelegationID,
)
if _, err := os.Stat(filepath.Join(exchangeRoot, "inputs", "input.txt")); err != nil {
t.Fatalf("failed exchange input not retained: %v", err)
}
if _, err := os.Stat(filepath.Join(workspace, ".delegations", captured.DelegationID)); !os.IsNotExist(err) {
t.Fatalf("failed exchange was published: %v", err)
}
}
type delegateTraceCaptureStore struct {
store.TracingStore
mu sync.Mutex
spans []store.SpanData
}
func (s *delegateTraceCaptureStore) BatchCreateSpans(_ context.Context, spans []store.SpanData) error {
s.mu.Lock()
defer s.mu.Unlock()
s.spans = append(s.spans, spans...)
return nil
}
func (s *delegateTraceCaptureStore) BatchUpdateTraceAggregates(_ context.Context, _ uuid.UUID) error {
return nil
}
func (s *delegateTraceCaptureStore) RecoverStaleRunningTraces(_ context.Context, _ time.Time) (int64, error) {
return 0, nil
}
func (s *delegateTraceCaptureStore) DeleteTracesOlderThan(_ context.Context, _ time.Time) (int64, error) {
return 0, nil
}
func TestDelegateTool_EmitsSanitizedArtifactLifecycleOnDelegateTrace(t *testing.T) {
workspace := t.TempDir()
traceStore := &delegateTraceCaptureStore{}
collector := tracing.NewCollector(traceStore)
collector.Start()
traceID := uuid.New()
runFn := func(_ context.Context, req DelegateRequest) (DelegateResult, error) {
if err := os.WriteFile(filepath.Join(req.DelegateOutputsPath, "result.json"), []byte("{}"), 0600); err != nil {
return DelegateResult{}, err
}
return DelegateResult{Content: "done", TraceID: traceID}, nil
}
tool := newDelegateTestTool(t, noopAgentLink{}, runFn)
ctx := tracing.WithCollector(WithToolWorkspace(makeDelegateCtx(t), workspace), collector)
result := tool.Execute(ctx, map[string]any{
"agent_key": "child-agent",
"task": "create result",
"mode": "sync",
})
if result == nil || result.IsError {
t.Fatalf("delegate result = %#v, want success", result)
}
collector.Stop()
traceStore.mu.Lock()
defer traceStore.mu.Unlock()
if len(traceStore.spans) != 3 {
t.Fatalf("spans = %#v, want staged, published, cleaned", traceStore.spans)
}
wantNames := []string{
"delegate.artifacts.staged",
"delegate.artifacts.published",
"delegate.artifacts.cleaned",
}
for i, span := range traceStore.spans {
if span.TraceID != traceID || span.SpanType != store.SpanTypeEvent ||
span.Name != wantNames[i] {
t.Fatalf("lifecycle span %d = %#v, want %s", i, span, wantNames[i])
}
metadata := string(span.Metadata)
if strings.Contains(metadata, workspace) {
t.Fatalf("lifecycle metadata leaked workspace: %s", metadata)
}
}
for _, index := range []int{1, 2} {
metadata := string(traceStore.spans[index].Metadata)
if !strings.Contains(metadata, `"path":"outputs/result.json"`) {
t.Fatalf("output lifecycle metadata = %s", metadata)
}
}
}
func TestDelegateTool_EmitsSanitizedFailedArtifactLifecycle(t *testing.T) {
workspace := t.TempDir()
if err := os.WriteFile(filepath.Join(workspace, "input.txt"), []byte("input"), 0600); err != nil {
t.Fatal(err)
}
traceStore := &delegateTraceCaptureStore{}
collector := tracing.NewCollector(traceStore)
collector.Start()
traceID := uuid.New()
runFn := func(_ context.Context, req DelegateRequest) (DelegateResult, error) {
if req.OnTraceCreated != nil {
req.OnTraceCreated(traceID)
}
return DelegateResult{TraceID: traceID}, fmt.Errorf(
"failed while using %s",
req.DelegateOutputsPath,
)
}
tool := newDelegateTestTool(t, noopAgentLink{}, runFn)
ctx := tracing.WithCollector(WithToolWorkspace(makeDelegateCtx(t), workspace), collector)
result := tool.Execute(ctx, map[string]any{
"agent_key": "child-agent",
"task": "fail safely",
"mode": "sync",
"inputs": []any{"input.txt"},
})
if result == nil || !result.IsError {
t.Fatalf("delegate result = %#v, want failure", result)
}
if strings.Contains(result.ForLLM, workspace) {
t.Fatalf("delegate error leaked workspace: %s", result.ForLLM)
}
collector.Stop()
traceStore.mu.Lock()
defer traceStore.mu.Unlock()
if len(traceStore.spans) != 2 {
t.Fatalf("spans = %#v, want staged and failed", traceStore.spans)
}
wantNames := []string{
"delegate.artifacts.staged",
"delegate.artifacts.failed",
}
for i, span := range traceStore.spans {
if span.TraceID != traceID || span.Name != wantNames[i] {
t.Fatalf("failure lifecycle span %d = %#v", i, span)
}
metadata := string(span.Metadata)
if strings.Contains(metadata, workspace) ||
!strings.Contains(metadata, `"path":"inputs/input.txt"`) {
t.Fatalf("failure lifecycle metadata = %s", metadata)
}
}
}
func TestDelegateTool_EmitsCancelledArtifactLifecycle(t *testing.T) {
traceStore := &delegateTraceCaptureStore{}
collector := tracing.NewCollector(traceStore)
collector.Start()
traceID := uuid.New()
runFn := func(_ context.Context, req DelegateRequest) (DelegateResult, error) {
if req.OnTraceCreated != nil {
req.OnTraceCreated(traceID)
}
return DelegateResult{TraceID: traceID}, context.Canceled
}
tool := newDelegateTestTool(t, noopAgentLink{}, runFn)
ctx := tracing.WithCollector(makeDelegateCtx(t), collector)
result := tool.Execute(ctx, map[string]any{
"agent_key": "child-agent",
"task": "cancel safely",
"mode": "sync",
})
if result == nil || !result.IsError {
t.Fatalf("delegate result = %#v, want cancellation", result)
}
collector.Stop()
traceStore.mu.Lock()
defer traceStore.mu.Unlock()
if len(traceStore.spans) != 2 ||
traceStore.spans[0].Name != "delegate.artifacts.staged" ||
traceStore.spans[1].Name != "delegate.artifacts.cancelled" {
t.Fatalf("cancellation lifecycle spans = %#v", traceStore.spans)
}
for _, span := range traceStore.spans {
if span.TraceID != traceID {
t.Fatalf("cancellation trace ID = %s, want %s", span.TraceID, traceID)
}
}
}
func TestDelegateTool_SweeperDeletesOnlyRegisteredExpiredExchange(t *testing.T) {
tenantWorkspace := t.TempDir()
delegationID := uuid.New()
exchangeRoot := filepath.Join(tenantWorkspace, "collaboration", "delegations", delegationID.String())
if err := os.MkdirAll(exchangeRoot, 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(exchangeRoot, "retained.txt"), []byte("retained"), 0600); err != nil {
t.Fatal(err)
}
outside := t.TempDir()
sentinel := filepath.Join(outside, "sentinel.txt")
if err := os.WriteFile(sentinel, []byte("keep"), 0600); err != nil {
t.Fatal(err)
}
if err := os.Symlink(outside, filepath.Join(exchangeRoot, "hostile-link")); err != nil {
t.Logf("symlink cleanup coverage unavailable: %v", err)
}
tool := NewDelegateTool(noopAgentLink{}, noopAgentCRUD{}, nil, func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
return DelegateResult{}, nil
})
item := retainedDelegationArtifact{
tenantWorkspace: tenantWorkspace,
tenantID: store.MasterTenantID,
delegationID: delegationID,
retainUntil: time.Now().Add(-time.Minute),
}
tool.retained[retainedDelegationArtifactKey(tenantWorkspace, delegationID)] = item
tool.sweepRetainedDelegationExchanges(time.Now())
if _, err := os.Stat(exchangeRoot); !os.IsNotExist(err) {
t.Fatalf("expired exchange still exists: %v", err)
}
if len(tool.retained) != 0 {
t.Fatalf("retained registry = %#v, want empty", tool.retained)
}
if got, err := os.ReadFile(sentinel); err != nil || string(got) != "keep" {
t.Fatalf("secure cleanup followed hostile link: %q, %v", got, err)
}
}
func TestDelegateTool_RecoversRetainedExchangeAfterRestart(t *testing.T) {
tenantWorkspace := t.TempDir()
callerWorkspace := filepath.Join(tenantWorkspace, "agents", "caller")
if err := os.MkdirAll(callerWorkspace, 0750); err != nil {
t.Fatal(err)
}
callerRoot, err := OpenDelegationArtifactRoot(callerWorkspace)
if err != nil {
t.Fatal(err)
}
delegationID := uuid.New()
exchange, err := NewDelegationArtifactExchange(
tenantWorkspace,
store.MasterTenantID,
delegationID,
DelegationArtifactLimits{},
time.Minute,
)
if err != nil {
t.Fatal(err)
}
first := NewDelegateTool(noopAgentLink{}, noopAgentCRUD{}, nil, func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
return DelegateResult{}, nil
})
first.SetWorkspace(tenantWorkspace)
job := &delegateArtifactJob{
req: DelegateRequest{DelegationID: delegationID.String()},
callerRoot: callerRoot,
callerWorkspace: callerWorkspace,
tenantWorkspace: tenantWorkspace,
tenantID: store.MasterTenantID,
delegationID: delegationID,
}
job.callerLocation = first.resolveDelegationCallerLocation(job)
exchange.RetainFailure(time.Now().Add(-2*time.Minute), "test_failure")
if err := first.registerRetainedDelegationExchange(exchange, job, artifactLifecycleFailed); err != nil {
t.Fatalf("register retained exchange: %v", err)
}
stateBytes, err := os.ReadFile(filepath.Join(
tenantWorkspace,
"collaboration",
"delegations",
delegationID.String(),
delegationArtifactLifecycleFile,
))
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(stateBytes), tenantWorkspace) ||
strings.Contains(string(stateBytes), callerWorkspace) {
t.Fatalf("lifecycle state persisted a host path: %s", stateBytes)
}
if err := exchange.Close(); err != nil {
t.Fatal(err)
}
if err := callerRoot.Close(); err != nil {
t.Fatal(err)
}
first.Close()
restarted := NewDelegateTool(noopAgentLink{}, noopAgentCRUD{}, nil, func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
return DelegateResult{}, nil
})
restarted.SetWorkspace(tenantWorkspace)
if len(restarted.retained) != 1 {
t.Fatalf("recovered retained registry = %#v, want one", restarted.retained)
}
restarted.sweepRetainedDelegationExchanges(time.Now())
restarted.Close()
exchangeRoot := filepath.Join(
tenantWorkspace,
"collaboration",
"delegations",
delegationID.String(),
)
if _, err := os.Stat(exchangeRoot); !os.IsNotExist(err) {
t.Fatalf("restart janitor left expired exchange: %v", err)
}
}
func TestDelegateTool_RecoversStaleArtifactLifecycleStates(t *testing.T) {
testCases := []struct {
name string
status delegationArtifactLifecycleStatus
}{
{name: "staging", status: artifactLifecycleStaging},
{name: "running", status: artifactLifecycleRunning},
{name: "publishing", status: artifactLifecyclePublishing},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
tenantWorkspace := t.TempDir()
callerWorkspace := filepath.Join(tenantWorkspace, "agents", "caller")
if err := os.MkdirAll(callerWorkspace, 0750); err != nil {
t.Fatal(err)
}
callerRoot, err := OpenDelegationArtifactRoot(callerWorkspace)
if err != nil {
t.Fatal(err)
}
delegationID := uuid.New()
exchange, err := NewDelegationArtifactExchange(
tenantWorkspace,
store.MasterTenantID,
delegationID,
DelegationArtifactLimits{},
time.Minute,
)
if err != nil {
t.Fatal(err)
}
first := NewDelegateTool(
noopAgentLink{},
noopAgentCRUD{},
nil,
func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
return DelegateResult{}, nil
},
)
first.workspace = tenantWorkspace
job := &delegateArtifactJob{
req: DelegateRequest{DelegationID: delegationID.String()},
callerRoot: callerRoot,
callerWorkspace: callerWorkspace,
tenantWorkspace: tenantWorkspace,
tenantID: store.MasterTenantID,
delegationID: delegationID,
}
job.callerLocation = first.resolveDelegationCallerLocation(job)
if err := first.updateActiveDelegationLifecycle(exchange, job); err != nil {
t.Fatalf("update staging lifecycle: %v", err)
}
publicationTempPath := ""
if testCase.status == artifactLifecycleRunning ||
testCase.status == artifactLifecyclePublishing {
if err := first.markDelegationRunning(exchange, job, time.Now()); err != nil {
t.Fatalf("mark running: %v", err)
}
}
if testCase.status == artifactLifecyclePublishing {
publicationTempPath = filepath.ToSlash(filepath.Join(
".delegations",
".tmp-"+delegationID.String()+"-"+uuid.NewString(),
))
if err := first.markDelegationPublishing(
exchange,
job,
publicationTempPath,
time.Now(),
); err != nil {
t.Fatalf("mark publishing: %v", err)
}
if err := os.MkdirAll(
filepath.Join(callerWorkspace, filepath.FromSlash(publicationTempPath)),
0700,
); err != nil {
t.Fatal(err)
}
}
finalSentinel := filepath.Join(
callerWorkspace,
".delegations",
delegationID.String(),
"keep.txt",
)
if err := os.MkdirAll(filepath.Dir(finalSentinel), 0750); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(finalSentinel, []byte("keep"), 0600); err != nil {
t.Fatal(err)
}
if err := exchange.Close(); err != nil {
t.Fatal(err)
}
if err := callerRoot.Close(); err != nil {
t.Fatal(err)
}
first.Close()
restarted := NewDelegateTool(
noopAgentLink{},
noopAgentCRUD{},
nil,
func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
return DelegateResult{}, nil
},
)
recoveredAfter := time.Now().UTC()
restarted.SetWorkspace(tenantWorkspace)
defer restarted.Close()
if len(restarted.retained) != 1 {
t.Fatalf("recovered registry = %#v, want one", restarted.retained)
}
lifecyclePath := filepath.Join(
tenantWorkspace,
"collaboration",
"delegations",
delegationID.String(),
delegationArtifactLifecycleFile,
)
stateBytes, err := os.ReadFile(lifecyclePath)
if err != nil {
t.Fatal(err)
}
var state delegationArtifactLifecycleState
if err := json.Unmarshal(stateBytes, &state); err != nil {
t.Fatal(err)
}
if state.Status != artifactLifecycleFailed ||
state.FailedAt == nil ||
state.ReasonCode != "artifact_recovered_stale" {
t.Fatalf("recovered state = %#v", state)
}
earliestExpiry := recoveredAfter.Add(delegationArtifactFailureTTL)
if state.RetainUntil.Before(earliestExpiry) ||
state.RetainUntil.After(time.Now().UTC().Add(delegationArtifactFailureTTL)) {
t.Fatalf("retain until = %v, want recovery time + %v", state.RetainUntil, delegationArtifactFailureTTL)
}
restarted.sweepRetainedDelegationExchanges(state.RetainUntil.Add(-time.Nanosecond))
if _, err := os.Stat(lifecyclePath); err != nil {
t.Fatalf("retained exchange removed before TTL: %v", err)
}
restarted.sweepRetainedDelegationExchanges(state.RetainUntil)
if _, err := os.Stat(filepath.Dir(lifecyclePath)); !os.IsNotExist(err) {
t.Fatalf("expired exchange remains: %v", err)
}
if publicationTempPath != "" {
if _, err := os.Stat(filepath.Join(
callerWorkspace,
filepath.FromSlash(publicationTempPath),
)); !os.IsNotExist(err) {
t.Fatalf("recorded publication temp remains: %v", err)
}
}
if got, err := os.ReadFile(finalSentinel); err != nil || string(got) != "keep" {
t.Fatalf("final UUID publication directory was modified: %q, %v", got, err)
}
})
}
}
func TestDelegateTool_PeriodicArtifactMaintenancePreservesActiveExchange(t *testing.T) {
tenantWorkspace := t.TempDir()
callerWorkspace := filepath.Join(tenantWorkspace, "agents", "caller")
if err := os.MkdirAll(callerWorkspace, 0o750); err != nil {
t.Fatal(err)
}
callerRoot, err := OpenDelegationArtifactRoot(callerWorkspace)
if err != nil {
t.Fatal(err)
}
defer callerRoot.Close()
delegationID := uuid.New()
runStarted := make(chan struct{})
resumeRun := make(chan struct{})
var tool *DelegateTool
tool = NewDelegateTool(
noopAgentLink{},
noopAgentCRUD{},
nil,
func(_ context.Context, req DelegateRequest) (DelegateResult, error) {
close(runStarted)
<-resumeRun
if !tool.delegationArtifactExchangeActive(tenantWorkspace, delegationID) {
return DelegateResult{}, errors.New("delegation artifact exchange became inactive during run")
}
if err := os.WriteFile(
filepath.Join(req.DelegateOutputsPath, "result.txt"),
[]byte("active result"),
0o600,
); err != nil {
return DelegateResult{}, err
}
return DelegateResult{Content: "done"}, nil
},
)
tool.workspace = tenantWorkspace
defer tool.Close()
job := &delegateArtifactJob{
req: DelegateRequest{DelegationID: delegationID.String()},
callerRoot: callerRoot,
callerWorkspace: callerWorkspace,
tenantWorkspace: tenantWorkspace,
tenantID: store.MasterTenantID,
delegationID: delegationID,
}
job.callerLocation = tool.resolveDelegationCallerLocation(job)
runDone := make(chan error, 1)
go func() {
_, err := tool.runArtifactExchange(context.Background(), job)
runDone <- err
}()
select {
case <-runStarted:
case <-time.After(2 * time.Second):
close(resumeRun)
t.Fatal("delegated run did not start")
}
tool.addRetainedDelegationArtifact(retainedDelegationArtifact{
tenantWorkspace: tenantWorkspace,
tenantID: store.MasterTenantID,
delegationID: uuid.New(),
retainUntil: time.Now().Add(time.Hour),
})
tool.maintainDelegationArtifacts(time.Now())
lifecyclePath := filepath.Join(
tenantWorkspace,
"collaboration",
"delegations",
delegationID.String(),
delegationArtifactLifecycleFile,
)
stateBytes, stateErr := os.ReadFile(lifecyclePath)
var state delegationArtifactLifecycleState
if stateErr == nil {
stateErr = json.Unmarshal(stateBytes, &state)
}
close(resumeRun)
runErr := <-runDone
if stateErr != nil {
t.Fatal(stateErr)
}
if state.Status != artifactLifecycleRunning {
t.Fatalf("active lifecycle status = %q, want %q", state.Status, artifactLifecycleRunning)
}
if runErr != nil {
t.Fatalf("active exchange run after maintenance: %v", runErr)
}
if tool.delegationArtifactExchangeActive(tenantWorkspace, delegationID) {
t.Fatal("completed delegation artifact exchange remained active")
}
publishedOutput := filepath.Join(
callerWorkspace,
".delegations",
delegationID.String(),
"outputs",
"result.txt",
)
if got, err := os.ReadFile(publishedOutput); err != nil || string(got) != "active result" {
t.Fatalf("published output = %q, %v", got, err)
}
}
func TestDelegateTool_PeriodicArtifactMaintenanceRecoversUnregisteredExchange(t *testing.T) {
tenantWorkspace := t.TempDir()
callerWorkspace := filepath.Join(tenantWorkspace, "agents", "caller")
if err := os.MkdirAll(callerWorkspace, 0o750); err != nil {
t.Fatal(err)
}
callerRoot, err := OpenDelegationArtifactRoot(callerWorkspace)
if err != nil {
t.Fatal(err)
}
defer callerRoot.Close()
delegationID := uuid.New()
exchange, err := NewDelegationArtifactExchange(
tenantWorkspace,
store.MasterTenantID,
delegationID,
DelegationArtifactLimits{},
time.Minute,
)
if err != nil {
t.Fatal(err)
}
defer exchange.Close()
tool := NewDelegateTool(
noopAgentLink{},
noopAgentCRUD{},
nil,
func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
return DelegateResult{}, nil
},
)
tool.workspace = tenantWorkspace
defer tool.Close()
job := &delegateArtifactJob{
req: DelegateRequest{DelegationID: delegationID.String()},
callerRoot: callerRoot,
callerWorkspace: callerWorkspace,
tenantWorkspace: tenantWorkspace,
tenantID: store.MasterTenantID,
delegationID: delegationID,
}
job.callerLocation = tool.resolveDelegationCallerLocation(job)
if err := tool.updateActiveDelegationLifecycle(exchange, job); err != nil {
t.Fatalf("update active lifecycle: %v", err)
}
if err := tool.markDelegationRunning(exchange, job, time.Now()); err != nil {
t.Fatalf("mark running: %v", err)
}
tool.maintainDelegationArtifacts(time.Now())
state, err := readDelegationArtifactLifecycleState(exchange.root, delegationID)
if err != nil {
t.Fatal(err)
}
if state.Status != artifactLifecycleFailed || state.ReasonCode != "artifact_recovered_stale" {
t.Fatalf("unregistered lifecycle = %#v, want recovered stale failure", state)
}
if _, ok := tool.retained[retainedDelegationArtifactKey(tenantWorkspace, delegationID)]; !ok {
t.Fatalf("unregistered exchange was not added to retained registry: %#v", tool.retained)
}
}
func TestDelegateTool_RecoveryPromotesDurablePublishingState(t *testing.T) {
tenantWorkspace := t.TempDir()
callerWorkspace := filepath.Join(tenantWorkspace, "agents", "caller")
if err := os.MkdirAll(callerWorkspace, 0o750); err != nil {
t.Fatal(err)
}
callerRoot, err := OpenDelegationArtifactRoot(callerWorkspace)
if err != nil {
t.Fatal(err)
}
delegationID := uuid.New()
exchange, err := NewDelegationArtifactExchange(
tenantWorkspace,
store.MasterTenantID,
delegationID,
DelegationArtifactLimits{},
time.Minute,
)
if err != nil {
t.Fatal(err)
}
first := NewDelegateTool(
noopAgentLink{},
noopAgentCRUD{},
nil,
func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
return DelegateResult{}, nil
},
)
first.workspace = tenantWorkspace
job := &delegateArtifactJob{
req: DelegateRequest{DelegationID: delegationID.String()},
callerRoot: callerRoot,
callerWorkspace: callerWorkspace,
tenantWorkspace: tenantWorkspace,
tenantID: store.MasterTenantID,
delegationID: delegationID,
}
job.callerLocation = first.resolveDelegationCallerLocation(job)
if err := first.updateActiveDelegationLifecycle(exchange, job); err != nil {
t.Fatalf("update lifecycle: %v", err)
}
if err := first.markDelegationRunning(exchange, job, time.Now()); err != nil {
t.Fatalf("mark running: %v", err)
}
if err := os.WriteFile(
filepath.Join(exchange.OutputsHostPath(), "result.txt"),
[]byte("durable result"),
0o600,
); err != nil {
t.Fatal(err)
}
publication, err := exchange.publishWithPreparation(
context.Background(),
callerRoot,
time.Now(),
func(tempPath string) error {
return first.markDelegationPublishing(exchange, job, tempPath, time.Now())
},
)
if err != nil {
t.Fatalf("publish before simulated crash: %v", err)
}
if err := exchange.Close(); err != nil {
t.Fatal(err)
}
if err := callerRoot.Close(); err != nil {
t.Fatal(err)
}
first.Close()
restarted := NewDelegateTool(
noopAgentLink{},
noopAgentCRUD{},
nil,
func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
return DelegateResult{}, nil
},
)
restarted.SetWorkspace(tenantWorkspace)
defer restarted.Close()
item, ok := restarted.retained[retainedDelegationArtifactKey(tenantWorkspace, delegationID)]
if !ok || !item.publicationDurable || !item.retainUntil.IsZero() {
t.Fatalf("recovered publication = %#v, want durable immediate cleanup", item)
}
lifecyclePath := filepath.Join(
tenantWorkspace,
"collaboration",
"delegations",
delegationID.String(),
delegationArtifactLifecycleFile,
)
stateBytes, err := os.ReadFile(lifecyclePath)
if err != nil {
t.Fatal(err)
}
var state delegationArtifactLifecycleState
if err := json.Unmarshal(stateBytes, &state); err != nil {
t.Fatal(err)
}
if state.Status != artifactLifecyclePublished || state.ReasonCode != "artifact_published" {
t.Fatalf("recovered lifecycle = %#v, want published", state)
}
restarted.sweepRetainedDelegationExchanges(time.Now())
if _, err := os.Stat(filepath.Dir(lifecyclePath)); !os.IsNotExist(err) {
t.Fatalf("published exchange remains after cleanup: %v", err)
}
publishedOutput := filepath.Join(
callerWorkspace,
filepath.FromSlash(publication.RootPath),
"outputs",
"result.txt",
)
if got, err := os.ReadFile(publishedOutput); err != nil || string(got) != "durable result" {
t.Fatalf("durable publication changed during recovery: %q, %v", got, err)
}
}
func TestDelegateTool_RecoveryCleansInvalidAndExpiresCorruptExchanges(t *testing.T) {
tenantWorkspace := t.TempDir()
delegationsRoot := filepath.Join(tenantWorkspace, "collaboration", "delegations")
invalidRoot := filepath.Join(delegationsRoot, "not-a-delegation")
if err := os.MkdirAll(invalidRoot, 0750); err != nil {
t.Fatal(err)
}
corruptID := uuid.New()
corruptRoot := filepath.Join(delegationsRoot, corruptID.String())
if err := os.MkdirAll(corruptRoot, 0750); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(
filepath.Join(corruptRoot, delegationArtifactLifecycleFile),
[]byte("{not-json"),
0600,
); err != nil {
t.Fatal(err)
}
tool := NewDelegateTool(
noopAgentLink{},
noopAgentCRUD{},
nil,
func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
return DelegateResult{}, nil
},
)
tool.SetWorkspace(tenantWorkspace)
defer tool.Close()
if _, err := os.Stat(invalidRoot); !os.IsNotExist(err) {
t.Fatalf("invalid exchange entry remains after recovery: %v", err)
}
item, ok := tool.retained[retainedDelegationArtifactKey(tenantWorkspace, corruptID)]
if !ok {
t.Fatalf("corrupt exchange was not registered for bounded retention: %#v", tool.retained)
}
tool.sweepRetainedDelegationExchanges(item.retainUntil.Add(-time.Nanosecond))
if _, err := os.Stat(corruptRoot); err != nil {
t.Fatalf("corrupt exchange removed before retention elapsed: %v", err)
}
tool.sweepRetainedDelegationExchanges(item.retainUntil)
if _, err := os.Stat(corruptRoot); !os.IsNotExist(err) {
t.Fatalf("corrupt exchange remains after retention elapsed: %v", err)
}
}
func TestDelegateTool_RecoversPublicationCleanupAfterRestart(t *testing.T) {
tenantWorkspace := t.TempDir()
callerWorkspace := filepath.Join(tenantWorkspace, "agents", "caller")
if err := os.MkdirAll(callerWorkspace, 0750); err != nil {
t.Fatal(err)
}
callerRoot, err := OpenDelegationArtifactRoot(callerWorkspace)
if err != nil {
t.Fatal(err)
}
delegationID := uuid.New()
exchange, err := NewDelegationArtifactExchange(
tenantWorkspace,
store.MasterTenantID,
delegationID,
DelegationArtifactLimits{},
time.Minute,
)
if err != nil {
t.Fatal(err)
}
first := NewDelegateTool(
noopAgentLink{},
noopAgentCRUD{},
nil,
func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
return DelegateResult{}, nil
},
)
first.workspace = tenantWorkspace
job := &delegateArtifactJob{
req: DelegateRequest{DelegationID: delegationID.String()},
callerRoot: callerRoot,
callerWorkspace: callerWorkspace,
tenantWorkspace: tenantWorkspace,
tenantID: store.MasterTenantID,
delegationID: delegationID,
}
job.callerLocation = first.resolveDelegationCallerLocation(job)
if err := first.updateActiveDelegationLifecycle(exchange, job); err != nil {
t.Fatalf("update staging lifecycle: %v", err)
}
if err := first.markDelegationRunning(exchange, job, time.Now()); err != nil {
t.Fatalf("mark running: %v", err)
}
if err := os.WriteFile(
filepath.Join(exchange.OutputsHostPath(), "result.txt"),
[]byte("durable"),
0600,
); err != nil {
t.Fatal(err)
}
var publicationTempPath string
publication, err := exchange.publishWithPreparation(
context.Background(),
callerRoot,
time.Now(),
func(tempPath string) error {
publicationTempPath = tempPath
return first.markDelegationPublishing(exchange, job, tempPath, time.Now())
},
)
if err != nil {
t.Fatalf("publish before simulated restart: %v", err)
}
if err := first.markDelegationPublished(exchange, job, publication.Manifest.PublishedAt); err != nil {
t.Fatalf("mark published: %v", err)
}
if publicationTempPath == "" {
t.Fatal("publication temp path was not recorded")
}
if err := exchange.Close(); err != nil {
t.Fatal(err)
}
if err := callerRoot.Close(); err != nil {
t.Fatal(err)
}
first.Close()
restarted := NewDelegateTool(
noopAgentLink{},
noopAgentCRUD{},
nil,
func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
return DelegateResult{}, nil
},
)
restarted.SetWorkspace(tenantWorkspace)
if len(restarted.retained) != 1 {
t.Fatalf("recovered cleanup registry = %#v, want one", restarted.retained)
}
restarted.sweepRetainedDelegationExchanges(time.Now())
restarted.Close()
exchangeRoot := filepath.Join(
tenantWorkspace,
"collaboration",
"delegations",
delegationID.String(),
)
if _, err := os.Stat(exchangeRoot); !os.IsNotExist(err) {
t.Fatalf("restart janitor left publication exchange: %v", err)
}
publicationRoot := filepath.Join(
callerWorkspace,
".delegations",
delegationID.String(),
)
published, err := os.ReadFile(filepath.Join(publicationRoot, "outputs", "result.txt"))
if err != nil || string(published) != "durable" {
t.Fatalf("restart janitor damaged durable publication: %q, %v", published, err)
}
}
func TestDelegateTool_RetriesPublishedExchangeCleanup(t *testing.T) {
managedWorkspace := t.TempDir()
callerWorkspace := t.TempDir()
traceStore := &delegateTraceCaptureStore{}
collector := tracing.NewCollector(traceStore)
collector.Start()
traceID := uuid.New()
var captured DelegateRequest
tool := NewDelegateTool(
noopAgentLink{},
noopAgentCRUD{},
nil,
func(_ context.Context, req DelegateRequest) (DelegateResult, error) {
captured = req
if err := os.WriteFile(
filepath.Join(req.DelegateOutputsPath, "result.txt"),
[]byte("durable"),
0600,
); err != nil {
return DelegateResult{}, err
}
return DelegateResult{Content: "done", TraceID: traceID}, nil
},
)
tool.SetWorkspace(managedWorkspace)
defer tool.Close()
removeCalls := 0
tool.removeExchange = func(tenantWorkspace string, delegationID uuid.UUID) error {
removeCalls++
if removeCalls == 1 {
return errors.New("injected cleanup failure")
}
return tool.removeDelegationExchange(tenantWorkspace, delegationID)
}
ctx := tracing.WithCollector(
WithToolWorkspace(makeDelegateCtx(t), callerWorkspace),
collector,
)
result := tool.Execute(ctx, map[string]any{
"agent_key": "child-agent",
"task": "publish output",
"mode": "sync",
})
if result == nil || result.IsError {
t.Fatalf("delegate result = %#v, want success", result)
}
delegationID := uuid.MustParse(captured.DelegationID)
exchangeRoot := filepath.Join(
managedWorkspace,
"collaboration",
"delegations",
delegationID.String(),
)
if _, err := os.Stat(exchangeRoot); err != nil {
t.Fatalf("failed cleanup did not retain exchange for retry: %v", err)
}
tool.retainedMu.Lock()
retainedCount := len(tool.retained)
sweeperStarted := tool.sweeperStarted
tool.retainedMu.Unlock()
if retainedCount != 1 || !sweeperStarted {
t.Fatalf("automatic cleanup registry = %d, sweeper started = %v", retainedCount, sweeperStarted)
}
tool.sweepRetainedDelegationExchanges(time.Now())
if removeCalls != 2 {
t.Fatalf("cleanup calls = %d, want immediate attempt plus retry", removeCalls)
}
if _, err := os.Stat(exchangeRoot); !os.IsNotExist(err) {
t.Fatalf("cleanup retry left exchange: %v", err)
}
published, err := os.ReadFile(filepath.Join(
callerWorkspace,
".delegations",
delegationID.String(),
"outputs",
"result.txt",
))
if err != nil || string(published) != "durable" {
t.Fatalf("cleanup retry damaged durable output: %q, %v", published, err)
}
collector.Stop()
traceStore.mu.Lock()
defer traceStore.mu.Unlock()
if len(traceStore.spans) != 3 ||
traceStore.spans[2].TraceID != traceID ||
traceStore.spans[2].Name != "delegate.artifacts.cleaned" {
t.Fatalf("cleanup retry spans = %#v, want cleaned on original trace", traceStore.spans)
}
}