mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-11 14:13:21 +00:00
2349 lines
71 KiB
Go
2349 lines
71 KiB
Go
package tools
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
|
|
"github.com/nextlevelbuilder/goclaw/internal/bus"
|
|
orchestration "github.com/nextlevelbuilder/goclaw/internal/childrun"
|
|
"github.com/nextlevelbuilder/goclaw/internal/config"
|
|
"github.com/nextlevelbuilder/goclaw/internal/eventbus"
|
|
"github.com/nextlevelbuilder/goclaw/internal/hooks"
|
|
"github.com/nextlevelbuilder/goclaw/internal/store"
|
|
"github.com/nextlevelbuilder/goclaw/internal/tracing"
|
|
)
|
|
|
|
// fakeDispatcher records Fire calls and returns a preset decision.
|
|
type fakeDispatcher struct {
|
|
decision hooks.Decision
|
|
calls int
|
|
}
|
|
|
|
type recordingDelegateEventBus struct {
|
|
mu sync.Mutex
|
|
events []eventbus.DomainEvent
|
|
}
|
|
|
|
func (b *recordingDelegateEventBus) Publish(event eventbus.DomainEvent) {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
b.events = append(b.events, event)
|
|
}
|
|
|
|
func (*recordingDelegateEventBus) Subscribe(
|
|
eventbus.EventType,
|
|
eventbus.DomainEventHandler,
|
|
) func() {
|
|
return func() {}
|
|
}
|
|
|
|
func (*recordingDelegateEventBus) Start(context.Context) {}
|
|
|
|
func (*recordingDelegateEventBus) Drain(time.Duration) error { return nil }
|
|
|
|
func (b *recordingDelegateEventBus) snapshot() []eventbus.DomainEvent {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
return append([]eventbus.DomainEvent(nil), b.events...)
|
|
}
|
|
|
|
func (f *fakeDispatcher) Fire(_ context.Context, _ hooks.Event) (hooks.FireResult, error) {
|
|
f.calls++
|
|
return hooks.FireResult{Decision: f.decision}, nil
|
|
}
|
|
|
|
// --- minimal noop stores to satisfy AgentLinkStore / AgentCRUDStore ---
|
|
|
|
type noopAgentLink struct{}
|
|
|
|
var noopAgentLinkID = uuid.MustParse("8d9c2f78-7914-4e4e-b75f-e6e22ea75921")
|
|
|
|
func (noopAgentLink) CreateLink(_ context.Context, _ *store.AgentLinkData) error { return nil }
|
|
func (noopAgentLink) DeleteLink(_ context.Context, _ uuid.UUID) error { return nil }
|
|
func (noopAgentLink) UpdateLink(_ context.Context, _ uuid.UUID, _ map[string]any) error {
|
|
return nil
|
|
}
|
|
func (noopAgentLink) GetLink(_ context.Context, _ uuid.UUID) (*store.AgentLinkData, error) {
|
|
return nil, nil
|
|
}
|
|
func (noopAgentLink) ListLinksFrom(_ context.Context, _ uuid.UUID) ([]store.AgentLinkData, error) {
|
|
return nil, nil
|
|
}
|
|
func (noopAgentLink) ListLinksTo(_ context.Context, _ uuid.UUID) ([]store.AgentLinkData, error) {
|
|
return nil, nil
|
|
}
|
|
func (noopAgentLink) CanDelegate(_ context.Context, _, _ uuid.UUID) (bool, error) {
|
|
return true, nil
|
|
}
|
|
func (noopAgentLink) GetLinkBetween(_ context.Context, _, _ uuid.UUID) (*store.AgentLinkData, error) {
|
|
return &store.AgentLinkData{
|
|
BaseModel: store.BaseModel{ID: noopAgentLinkID},
|
|
MaxConcurrent: 3,
|
|
Status: store.LinkStatusActive,
|
|
}, nil
|
|
}
|
|
func (noopAgentLink) DelegateTargets(_ context.Context, _ uuid.UUID) ([]store.AgentLinkData, error) {
|
|
return nil, nil
|
|
}
|
|
func (noopAgentLink) SearchDelegateTargets(_ context.Context, _ uuid.UUID, _ string, _ int) ([]store.AgentLinkData, error) {
|
|
return nil, nil
|
|
}
|
|
func (noopAgentLink) SearchDelegateTargetsByEmbedding(_ context.Context, _ uuid.UUID, _ []float32, _ int) ([]store.AgentLinkData, error) {
|
|
return nil, nil
|
|
}
|
|
func (noopAgentLink) DeleteTeamLinksForAgent(_ context.Context, _, _ uuid.UUID) error { return nil }
|
|
|
|
type fixedAgentLink struct {
|
|
noopAgentLink
|
|
link store.AgentLinkData
|
|
}
|
|
|
|
func (f fixedAgentLink) GetLinkBetween(_ context.Context, _, _ uuid.UUID) (*store.AgentLinkData, error) {
|
|
link := f.link
|
|
return &link, nil
|
|
}
|
|
|
|
type noopAgentCRUD struct {
|
|
keyToID map[string]uuid.UUID
|
|
}
|
|
|
|
func (n noopAgentCRUD) Create(_ context.Context, _ *store.AgentData) error { return nil }
|
|
func (n noopAgentCRUD) GetByKey(_ context.Context, key string) (*store.AgentData, error) {
|
|
id, ok := n.keyToID[key]
|
|
if !ok {
|
|
id = uuid.New()
|
|
}
|
|
return &store.AgentData{BaseModel: store.BaseModel{ID: id}, AgentKey: key}, nil
|
|
}
|
|
func (n noopAgentCRUD) GetByID(_ context.Context, _ uuid.UUID) (*store.AgentData, error) {
|
|
return nil, nil
|
|
}
|
|
func (n noopAgentCRUD) GetByIDUnscoped(_ context.Context, _ uuid.UUID) (*store.AgentData, error) {
|
|
return nil, nil
|
|
}
|
|
func (n noopAgentCRUD) GetByKeys(_ context.Context, _ []string) ([]store.AgentData, error) {
|
|
return nil, nil
|
|
}
|
|
func (n noopAgentCRUD) GetByIDs(_ context.Context, _ []uuid.UUID) ([]store.AgentData, error) {
|
|
return nil, nil
|
|
}
|
|
func (n noopAgentCRUD) Update(_ context.Context, _ uuid.UUID, _ map[string]any) error { return nil }
|
|
func (n noopAgentCRUD) Delete(_ context.Context, _ uuid.UUID) error { return nil }
|
|
func (n noopAgentCRUD) List(_ context.Context, _ string) ([]store.AgentData, error) { return nil, nil }
|
|
func (n noopAgentCRUD) GetDefault(_ context.Context) (*store.AgentData, error) { return nil, nil }
|
|
func (n noopAgentCRUD) ResetStuckSummoning(_ context.Context) (int64, error) { return 0, nil }
|
|
|
|
// --- helpers ---
|
|
|
|
func makeDelegateCtx(t *testing.T) context.Context {
|
|
t.Helper()
|
|
ctx := store.WithAgentID(context.Background(), uuid.New())
|
|
ctx = store.WithTenantID(ctx, store.MasterTenantID)
|
|
ctx = store.WithAgentKey(ctx, "parent-agent")
|
|
ctx = WithToolWorkspace(ctx, t.TempDir())
|
|
return ctx
|
|
}
|
|
|
|
func newDelegateTestTool(
|
|
t *testing.T,
|
|
links store.AgentLinkStore,
|
|
runFn DelegateRunFunc,
|
|
) *DelegateTool {
|
|
t.Helper()
|
|
tool := NewDelegateTool(links, noopAgentCRUD{}, nil, runFn)
|
|
tool.SetWorkspace(t.TempDir())
|
|
t.Cleanup(tool.Close)
|
|
return tool
|
|
}
|
|
|
|
// --- tests ---
|
|
|
|
func TestDelegateTool_EmitsSentOnlyAfterAdmissionWithLogicalTask(t *testing.T) {
|
|
t.Run("rejected admission emits nothing", func(t *testing.T) {
|
|
admission := orchestration.NewChildRunAdmission(1, 1)
|
|
if err := admission.Close(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
events := &recordingDelegateEventBus{}
|
|
tool := NewDelegateToolWithAdmission(
|
|
noopAgentLink{},
|
|
noopAgentCRUD{},
|
|
events,
|
|
func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
|
|
t.Fatal("runFn called after admission closed")
|
|
return DelegateResult{}, nil
|
|
},
|
|
admission,
|
|
)
|
|
tool.SetWorkspace(t.TempDir())
|
|
defer tool.Close()
|
|
|
|
result := tool.Execute(makeDelegateCtx(t), map[string]any{
|
|
"agent_key": "child-agent",
|
|
"task": "do something",
|
|
"mode": "sync",
|
|
})
|
|
if result == nil || !result.IsError {
|
|
t.Fatalf("result = %#v, want admission error", result)
|
|
}
|
|
if got := events.snapshot(); len(got) != 0 {
|
|
t.Fatalf("events = %#v, want none", got)
|
|
}
|
|
})
|
|
|
|
t.Run("accepted dispatch redacts caller host path", func(t *testing.T) {
|
|
events := &recordingDelegateEventBus{}
|
|
managedWorkspace := t.TempDir()
|
|
callerWorkspace := t.TempDir()
|
|
tool := NewDelegateTool(
|
|
noopAgentLink{},
|
|
noopAgentCRUD{},
|
|
events,
|
|
func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
|
|
return DelegateResult{Content: "done"}, nil
|
|
},
|
|
)
|
|
tool.SetWorkspace(managedWorkspace)
|
|
defer tool.Close()
|
|
|
|
ctx := WithToolWorkspace(makeDelegateCtx(t), callerWorkspace)
|
|
result := tool.Execute(ctx, map[string]any{
|
|
"agent_key": "child-agent",
|
|
"task": "inspect " + filepath.Join(callerWorkspace, "report.txt"),
|
|
"mode": "sync",
|
|
})
|
|
if result == nil || result.IsError {
|
|
t.Fatalf("result = %#v, want success", result)
|
|
}
|
|
eventsSnapshot := events.snapshot()
|
|
if len(eventsSnapshot) == 0 {
|
|
t.Fatal("delegate.sent not emitted")
|
|
}
|
|
var sent *eventbus.DelegateSentPayload
|
|
for _, event := range eventsSnapshot {
|
|
if event.Type != eventbus.EventDelegateSent {
|
|
continue
|
|
}
|
|
payload, ok := event.Payload.(eventbus.DelegateSentPayload)
|
|
if !ok {
|
|
t.Fatalf("delegate.sent payload = %T", event.Payload)
|
|
}
|
|
sent = &payload
|
|
break
|
|
}
|
|
if sent == nil {
|
|
t.Fatalf("events = %#v, want delegate.sent", eventsSnapshot)
|
|
}
|
|
if strings.Contains(sent.Task, callerWorkspace) ||
|
|
!strings.Contains(sent.Task, "caller workspace") {
|
|
t.Fatalf("delegate.sent task = %q", sent.Task)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestDelegateTool_SyncQueuedTimeoutDoesNotReportCompletion(t *testing.T) {
|
|
admission := orchestration.NewChildRunAdmission(1, 4)
|
|
blockerStarted := make(chan struct{})
|
|
blockerRelease := make(chan struct{})
|
|
blocker, err := admission.Enqueue(
|
|
context.Background(),
|
|
orchestration.ChildRunConstraints{TaskID: "blocker"},
|
|
func(context.Context, *orchestration.ChildRunLease) {
|
|
close(blockerStarted)
|
|
<-blockerRelease
|
|
},
|
|
)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := blocker.Activate(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
select {
|
|
case <-blockerStarted:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("blocker did not start")
|
|
}
|
|
|
|
events := &recordingDelegateEventBus{}
|
|
runCalled := false
|
|
tool := NewDelegateToolWithAdmission(
|
|
noopAgentLink{},
|
|
noopAgentCRUD{},
|
|
events,
|
|
func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
|
|
runCalled = true
|
|
return DelegateResult{Content: "unexpected"}, nil
|
|
},
|
|
admission,
|
|
)
|
|
tool.SetWorkspace(t.TempDir())
|
|
defer tool.Close()
|
|
ctx, cancel := context.WithTimeout(makeDelegateCtx(t), 20*time.Millisecond)
|
|
defer cancel()
|
|
result := tool.Execute(ctx, map[string]any{
|
|
"agent_key": "child-agent",
|
|
"task": "must time out while queued",
|
|
"mode": "sync",
|
|
})
|
|
if result == nil || !result.IsError {
|
|
t.Fatalf("result = %#v, want queued timeout", result)
|
|
}
|
|
if runCalled {
|
|
t.Fatal("queued timed-out delegation ran")
|
|
}
|
|
for _, event := range events.snapshot() {
|
|
if event.Type == eventbus.EventDelegateSent ||
|
|
event.Type == eventbus.EventDelegateCompleted {
|
|
t.Fatalf("timed-out queued delegation emitted %q", event.Type)
|
|
}
|
|
}
|
|
|
|
close(blockerRelease)
|
|
<-blocker.Done()
|
|
if err := admission.Close(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_SyncCallbackPanicReportsFailure(t *testing.T) {
|
|
events := &recordingDelegateEventBus{}
|
|
tool := NewDelegateTool(
|
|
noopAgentLink{},
|
|
noopAgentCRUD{},
|
|
events,
|
|
func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
|
|
panic("injected delegate panic")
|
|
},
|
|
)
|
|
tool.SetWorkspace(t.TempDir())
|
|
defer tool.Close()
|
|
result := tool.Execute(makeDelegateCtx(t), map[string]any{
|
|
"agent_key": "child-agent",
|
|
"task": "panic safely",
|
|
"mode": "sync",
|
|
})
|
|
if result == nil || !result.IsError ||
|
|
!strings.Contains(result.ForLLM, "child run panic") {
|
|
t.Fatalf("result = %#v, want recovered callback panic", result)
|
|
}
|
|
var failed, completed bool
|
|
for _, event := range events.snapshot() {
|
|
failed = failed || event.Type == eventbus.EventDelegateFailed
|
|
completed = completed || event.Type == eventbus.EventDelegateCompleted
|
|
}
|
|
if !failed || completed {
|
|
t.Fatalf("events = %#v, want failed without completed", events.snapshot())
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_SubagentStartBlock_AbortsDispatch(t *testing.T) {
|
|
runCalled := 0
|
|
runFn := func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
|
|
runCalled++
|
|
return DelegateResult{Content: "ok"}, nil
|
|
}
|
|
|
|
tool := newDelegateTestTool(t, noopAgentLink{}, runFn)
|
|
disp := &fakeDispatcher{decision: hooks.DecisionBlock}
|
|
tool.SetHookDispatcher(disp)
|
|
|
|
result := tool.Execute(makeDelegateCtx(t), map[string]any{
|
|
"agent_key": "child-agent",
|
|
"task": "do something",
|
|
"mode": "sync",
|
|
})
|
|
|
|
if result == nil || !result.IsError {
|
|
t.Fatal("expected error result when hook blocks")
|
|
}
|
|
if runCalled != 0 {
|
|
t.Errorf("runFn must not be called; got %d calls", runCalled)
|
|
}
|
|
if disp.calls != 1 {
|
|
t.Errorf("expected 1 dispatcher Fire call; got %d", disp.calls)
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_SubagentStartAllow_ProceedsToRun(t *testing.T) {
|
|
runCalled := 0
|
|
runFn := func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
|
|
runCalled++
|
|
return DelegateResult{Content: "done"}, nil
|
|
}
|
|
|
|
tool := newDelegateTestTool(t, noopAgentLink{}, runFn)
|
|
disp := &fakeDispatcher{decision: hooks.DecisionAllow}
|
|
tool.SetHookDispatcher(disp)
|
|
|
|
result := tool.Execute(makeDelegateCtx(t), map[string]any{
|
|
"agent_key": "child-agent",
|
|
"task": "do something",
|
|
"mode": "sync",
|
|
})
|
|
|
|
if result != nil && result.IsError {
|
|
t.Fatalf("unexpected error: %s", result.ForLLM)
|
|
}
|
|
if runCalled != 1 {
|
|
t.Errorf("expected runFn called once; got %d", runCalled)
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_NilDispatcher_SkipsHook(t *testing.T) {
|
|
runCalled := 0
|
|
runFn := func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
|
|
runCalled++
|
|
return DelegateResult{Content: "done"}, nil
|
|
}
|
|
|
|
// No SetHookDispatcher — hookDispatcher stays nil.
|
|
tool := newDelegateTestTool(t, noopAgentLink{}, runFn)
|
|
|
|
result := tool.Execute(makeDelegateCtx(t), map[string]any{
|
|
"agent_key": "child-agent",
|
|
"task": "do something",
|
|
"mode": "sync",
|
|
})
|
|
|
|
if result != nil && result.IsError {
|
|
t.Fatalf("unexpected error: %s", result.ForLLM)
|
|
}
|
|
if runCalled != 1 {
|
|
t.Errorf("expected runFn called once; got %d", runCalled)
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_SyncNestedRunCompletesBeforeOuterPublication(t *testing.T) {
|
|
admission := orchestration.NewChildRunAdmission(1, 8)
|
|
t.Cleanup(func() {
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
|
defer cancel()
|
|
if err := admission.Close(ctx); err != nil {
|
|
t.Errorf("close admission: %v", err)
|
|
}
|
|
})
|
|
|
|
callerWorkspace := t.TempDir()
|
|
managedWorkspace := t.TempDir()
|
|
childB := uuid.New()
|
|
childCStarted := make(chan struct{})
|
|
releaseChildC := make(chan struct{})
|
|
var childCCompleted atomic.Bool
|
|
|
|
var tool *DelegateTool
|
|
runFn := func(ctx context.Context, req DelegateRequest) (DelegateResult, error) {
|
|
switch req.ToAgentKey {
|
|
case "child-b":
|
|
nestedCtx := store.WithAgentID(ctx, childB)
|
|
nestedCtx = store.WithAgentKey(nestedCtx, "child-b")
|
|
nestedCtx = WithToolAgentKey(nestedCtx, "child-b")
|
|
nestedCtx = WithToolWorkspace(nestedCtx, req.DelegateOutputsPath)
|
|
nestedCtx = WithDelegationID(nestedCtx, req.DelegationID)
|
|
nestedCtx = WithDelegationArtifactInputs(nestedCtx, req.DelegateInputsPath)
|
|
nestedCtx = WithSubagentConfig(nestedCtx, &config.SubagentsConfig{MaxSpawnDepth: 1})
|
|
if depth := subagentDepthFromContext(nestedCtx, -1); depth != 0 {
|
|
return DelegateResult{}, fmt.Errorf("delegated agent depth = %d, want 0", depth)
|
|
}
|
|
if parentTaskID := subagentTaskIDFromContext(nestedCtx); parentTaskID != "" {
|
|
return DelegateResult{}, fmt.Errorf("delegated agent parent task = %q, want empty", parentTaskID)
|
|
}
|
|
if scope := subagentScopeFromContext(nestedCtx); scope.RootAgentID != childB ||
|
|
scope.RootAgentKey != "child-b" {
|
|
return DelegateResult{}, fmt.Errorf("delegated agent scope = %#v, want child-b", scope)
|
|
}
|
|
|
|
nested := tool.Execute(nestedCtx, map[string]any{
|
|
"agent_key": "child-c",
|
|
"task": "finish before child B returns",
|
|
"mode": "sync",
|
|
})
|
|
if nested == nil || nested.IsError {
|
|
return DelegateResult{}, fmt.Errorf("nested delegation failed: %#v", nested)
|
|
}
|
|
if !childCCompleted.Load() {
|
|
return DelegateResult{}, errors.New("nested delegation returned before child C completed")
|
|
}
|
|
if err := os.WriteFile(
|
|
filepath.Join(req.DelegateOutputsPath, "b-after-c.txt"),
|
|
[]byte("child C completed"),
|
|
0600,
|
|
); err != nil {
|
|
return DelegateResult{}, err
|
|
}
|
|
return DelegateResult{Content: "child B completed after child C"}, nil
|
|
|
|
case "child-c":
|
|
close(childCStarted)
|
|
select {
|
|
case <-releaseChildC:
|
|
case <-ctx.Done():
|
|
return DelegateResult{}, ctx.Err()
|
|
}
|
|
childCCompleted.Store(true)
|
|
return DelegateResult{Content: "child C completed"}, nil
|
|
|
|
default:
|
|
return DelegateResult{}, fmt.Errorf("unexpected target %q", req.ToAgentKey)
|
|
}
|
|
}
|
|
|
|
tool = NewDelegateToolWithAdmission(
|
|
noopAgentLink{},
|
|
noopAgentCRUD{keyToID: map[string]uuid.UUID{
|
|
"child-b": childB,
|
|
"child-c": uuid.New(),
|
|
}},
|
|
nil,
|
|
runFn,
|
|
admission,
|
|
)
|
|
tool.SetWorkspace(managedWorkspace)
|
|
t.Cleanup(tool.Close)
|
|
|
|
ctx := WithToolWorkspace(makeDelegateCtx(t), callerWorkspace)
|
|
ctx = WithSubagentConfig(ctx, &config.SubagentsConfig{MaxSpawnDepth: 1})
|
|
resultCh := make(chan *Result, 1)
|
|
go func() {
|
|
resultCh <- tool.Execute(ctx, map[string]any{
|
|
"agent_key": "child-b",
|
|
"task": "delegate synchronously to child C",
|
|
"mode": "sync",
|
|
})
|
|
}()
|
|
|
|
select {
|
|
case <-childCStarted:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("nested child C did not start")
|
|
}
|
|
select {
|
|
case result := <-resultCh:
|
|
t.Fatalf("outer delegation published before nested child completed: %#v", result)
|
|
case <-time.After(50 * time.Millisecond):
|
|
}
|
|
if entries, err := os.ReadDir(filepath.Join(callerWorkspace, ".delegations")); err == nil && len(entries) > 0 {
|
|
t.Fatalf("outer artifacts visible while nested child was active: %#v", entries)
|
|
} else if err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
t.Fatalf("inspect caller publication root: %v", err)
|
|
}
|
|
|
|
close(releaseChildC)
|
|
var result *Result
|
|
select {
|
|
case result = <-resultCh:
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("outer delegation did not finish after nested child completed")
|
|
}
|
|
if result == nil || result.IsError {
|
|
t.Fatalf("outer delegation = %#v, want success", result)
|
|
}
|
|
foundMarker := false
|
|
for _, media := range result.Media {
|
|
if media.Filename == "b-after-c.txt" {
|
|
foundMarker = true
|
|
content, err := os.ReadFile(media.Path)
|
|
if err != nil {
|
|
t.Fatalf("read published marker: %v", err)
|
|
}
|
|
if string(content) != "child C completed" {
|
|
t.Fatalf("published marker = %q", content)
|
|
}
|
|
}
|
|
}
|
|
if !foundMarker {
|
|
t.Fatalf("published media = %#v, want b-after-c.txt", result.Media)
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_ConcurrentMultiLinkTopologiesPublishInCallerScope(t *testing.T) {
|
|
managedWorkspace := t.TempDir()
|
|
callerA := t.TempDir()
|
|
callerC := t.TempDir()
|
|
agentA := uuid.New()
|
|
agentB := uuid.New()
|
|
agentC := uuid.New()
|
|
|
|
started := make(chan struct{}, 3)
|
|
release := make(chan struct{})
|
|
runFn := func(ctx context.Context, req DelegateRequest) (DelegateResult, error) {
|
|
started <- struct{}{}
|
|
select {
|
|
case <-release:
|
|
case <-ctx.Done():
|
|
return DelegateResult{}, ctx.Err()
|
|
}
|
|
if err := os.WriteFile(
|
|
filepath.Join(req.DelegateOutputsPath, "result.txt"),
|
|
[]byte(req.Task),
|
|
0600,
|
|
); err != nil {
|
|
return DelegateResult{}, err
|
|
}
|
|
return DelegateResult{Content: req.Task}, nil
|
|
}
|
|
tool := NewDelegateTool(
|
|
noopAgentLink{},
|
|
noopAgentCRUD{keyToID: map[string]uuid.UUID{
|
|
"agent-b": agentB,
|
|
"agent-c": agentC,
|
|
}},
|
|
nil,
|
|
runFn,
|
|
)
|
|
tool.SetWorkspace(managedWorkspace)
|
|
t.Cleanup(tool.Close)
|
|
|
|
type topologyCase struct {
|
|
name string
|
|
fromID uuid.UUID
|
|
fromKey string
|
|
toKey string
|
|
callerWorkspace string
|
|
payload string
|
|
}
|
|
cases := []topologyCase{
|
|
{
|
|
name: "a-to-b",
|
|
fromID: agentA,
|
|
fromKey: "agent-a",
|
|
toKey: "agent-b",
|
|
callerWorkspace: callerA,
|
|
payload: "A to B",
|
|
},
|
|
{
|
|
name: "a-to-c",
|
|
fromID: agentA,
|
|
fromKey: "agent-a",
|
|
toKey: "agent-c",
|
|
callerWorkspace: callerA,
|
|
payload: "A to C",
|
|
},
|
|
{
|
|
name: "c-to-b",
|
|
fromID: agentC,
|
|
fromKey: "agent-c",
|
|
toKey: "agent-b",
|
|
callerWorkspace: callerC,
|
|
payload: "C to B",
|
|
},
|
|
}
|
|
|
|
results := make([]*Result, len(cases))
|
|
var wg sync.WaitGroup
|
|
for i := range cases {
|
|
wg.Add(1)
|
|
go func(i int) {
|
|
defer wg.Done()
|
|
tc := cases[i]
|
|
ctx := store.WithAgentID(context.Background(), tc.fromID)
|
|
ctx = store.WithAgentKey(ctx, tc.fromKey)
|
|
ctx = store.WithTenantID(ctx, store.MasterTenantID)
|
|
ctx = WithToolWorkspace(ctx, tc.callerWorkspace)
|
|
results[i] = tool.Execute(ctx, map[string]any{
|
|
"agent_key": tc.toKey,
|
|
"task": tc.payload,
|
|
"mode": "sync",
|
|
})
|
|
}(i)
|
|
}
|
|
for range cases {
|
|
select {
|
|
case <-started:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("concurrent delegation did not start")
|
|
}
|
|
}
|
|
close(release)
|
|
wg.Wait()
|
|
|
|
aPublications := make(map[string]struct{})
|
|
for i, tc := range cases {
|
|
result := results[i]
|
|
if result == nil || result.IsError {
|
|
t.Fatalf("%s result = %#v", tc.name, result)
|
|
}
|
|
if len(result.Media) != 1 {
|
|
t.Fatalf("%s media = %#v, want one output", tc.name, result.Media)
|
|
}
|
|
media := result.Media[0]
|
|
relative, err := filepath.Rel(tc.callerWorkspace, media.Path)
|
|
if err != nil || strings.HasPrefix(relative, "..") {
|
|
t.Fatalf("%s output %q escaped caller %q", tc.name, media.Path, tc.callerWorkspace)
|
|
}
|
|
parts := strings.Split(filepath.ToSlash(relative), "/")
|
|
if len(parts) != 4 || parts[0] != ".delegations" || parts[2] != "outputs" || parts[3] != "result.txt" {
|
|
t.Fatalf("%s relative output = %q", tc.name, relative)
|
|
}
|
|
content, err := os.ReadFile(media.Path)
|
|
if err != nil {
|
|
t.Fatalf("%s read output: %v", tc.name, err)
|
|
}
|
|
if string(content) != tc.payload {
|
|
t.Fatalf("%s output = %q, want %q", tc.name, content, tc.payload)
|
|
}
|
|
if tc.fromKey == "agent-a" {
|
|
if _, exists := aPublications[parts[1]]; exists {
|
|
t.Fatalf("A→B and A→C reused delegation %q", parts[1])
|
|
}
|
|
aPublications[parts[1]] = struct{}{}
|
|
}
|
|
}
|
|
if len(aPublications) != 2 {
|
|
t.Fatalf("A publications = %d, want 2 isolated delegation roots", len(aPublications))
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_LinkMaxConcurrentIsReservedAndUnenforced(t *testing.T) {
|
|
linkID := uuid.New()
|
|
started := make(chan struct{})
|
|
releaseRun := make(chan struct{})
|
|
var calls atomic.Int32
|
|
runFn := func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
|
|
call := calls.Add(1)
|
|
if call == 1 {
|
|
close(started)
|
|
}
|
|
if call <= 2 {
|
|
<-releaseRun
|
|
}
|
|
return DelegateResult{Content: "done"}, nil
|
|
}
|
|
tool := newDelegateTestTool(t, fixedAgentLink{link: store.AgentLinkData{
|
|
BaseModel: store.BaseModel{ID: linkID},
|
|
MaxConcurrent: 1,
|
|
Status: store.LinkStatusActive,
|
|
}}, runFn)
|
|
|
|
first := tool.Execute(makeDelegateCtx(t), map[string]any{
|
|
"agent_key": "child-agent",
|
|
"task": "first",
|
|
"mode": "async",
|
|
})
|
|
if first == nil || first.IsError {
|
|
t.Fatalf("first delegation = %#v, want accepted", first)
|
|
}
|
|
select {
|
|
case <-started:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("first async delegation did not start")
|
|
}
|
|
|
|
second := tool.Execute(makeDelegateCtx(t), map[string]any{
|
|
"agent_key": "child-agent",
|
|
"task": "second",
|
|
"mode": "async",
|
|
})
|
|
if second == nil || second.IsError {
|
|
t.Fatalf("second delegation = %#v, want shared admission to accept it", second)
|
|
}
|
|
deadline := time.Now().Add(time.Second)
|
|
for calls.Load() < 2 && time.Now().Before(deadline) {
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
if got := calls.Load(); got != 2 {
|
|
t.Fatalf("runFn calls = %d, want max_concurrent ignored", got)
|
|
}
|
|
|
|
close(releaseRun)
|
|
|
|
third := tool.Execute(makeDelegateCtx(t), map[string]any{
|
|
"agent_key": "child-agent",
|
|
"task": "third",
|
|
"mode": "sync",
|
|
})
|
|
if third == nil || third.IsError {
|
|
t.Fatalf("third delegation = %#v, want success after release", third)
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_LinkSlotReleasedOnHookBlockAndSyncError(t *testing.T) {
|
|
link := fixedAgentLink{link: store.AgentLinkData{
|
|
BaseModel: store.BaseModel{ID: uuid.New()},
|
|
MaxConcurrent: 1,
|
|
Status: store.LinkStatusActive,
|
|
}}
|
|
var calls atomic.Int32
|
|
runFn := func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
|
|
if calls.Add(1) == 1 {
|
|
return DelegateResult{}, errors.New("first run failed")
|
|
}
|
|
return DelegateResult{Content: "done"}, nil
|
|
}
|
|
tool := newDelegateTestTool(t, link, runFn)
|
|
disp := &fakeDispatcher{decision: hooks.DecisionBlock}
|
|
tool.SetHookDispatcher(disp)
|
|
|
|
blocked := tool.Execute(makeDelegateCtx(t), map[string]any{
|
|
"agent_key": "child-agent",
|
|
"task": "blocked",
|
|
"mode": "sync",
|
|
})
|
|
if blocked == nil || !blocked.IsError {
|
|
t.Fatalf("blocked delegation = %#v, want error", blocked)
|
|
}
|
|
|
|
disp.decision = hooks.DecisionAllow
|
|
failed := tool.Execute(makeDelegateCtx(t), map[string]any{
|
|
"agent_key": "child-agent",
|
|
"task": "fails",
|
|
"mode": "sync",
|
|
})
|
|
if failed == nil || !failed.IsError ||
|
|
strings.Contains(failed.ForLLM, "active delegation limit reached") {
|
|
t.Fatalf("failed delegation = %#v, want run error after hook slot release", failed)
|
|
}
|
|
|
|
succeeded := tool.Execute(makeDelegateCtx(t), map[string]any{
|
|
"agent_key": "child-agent",
|
|
"task": "succeeds",
|
|
"mode": "sync",
|
|
})
|
|
if succeeded == nil || succeeded.IsError {
|
|
t.Fatalf("delegation after sync error = %#v, want success", succeeded)
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_PreservesGroupAuthorizationScope(t *testing.T) {
|
|
var captured DelegateRequest
|
|
runFn := func(_ context.Context, req DelegateRequest) (DelegateResult, error) {
|
|
captured = req
|
|
return DelegateResult{Content: "done"}, nil
|
|
}
|
|
tool := newDelegateTestTool(t, noopAgentLink{}, runFn)
|
|
|
|
ctx := makeDelegateCtx(t)
|
|
ctx = store.WithUserID(ctx, "group:telegram:-100123")
|
|
ctx = store.WithSenderID(ctx, "386246614")
|
|
ctx = store.WithRole(ctx, "viewer")
|
|
ctx = WithToolChannel(ctx, "telegram-main")
|
|
ctx = WithToolChannelType(ctx, "telegram")
|
|
ctx = WithToolChatID(ctx, "-100123")
|
|
ctx = WithToolPeerKind(ctx, "group")
|
|
|
|
result := tool.Execute(ctx, map[string]any{
|
|
"agent_key": "child-agent",
|
|
"task": "create a file",
|
|
"mode": "sync",
|
|
})
|
|
|
|
if result == nil || result.IsError {
|
|
t.Fatalf("delegate result = %#v, want success", result)
|
|
}
|
|
if captured.UserID != "group:telegram:-100123" {
|
|
t.Fatalf("UserID = %q, want group authorization scope", captured.UserID)
|
|
}
|
|
if captured.SenderID != "386246614" || captured.Role != "viewer" {
|
|
t.Fatalf("actor scope = (%q, %q), want real sender and role", captured.SenderID, captured.Role)
|
|
}
|
|
if captured.Channel != "telegram-main" || captured.ChannelType != "telegram" ||
|
|
captured.ChatID != "-100123" || captured.PeerKind != "group" {
|
|
t.Fatalf("origin scope = %#v", captured)
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_CurrentRunMediaBecomesReadOnlyRelativeInput(t *testing.T) {
|
|
workspace := t.TempDir()
|
|
source := filepath.Join(workspace, "reference.jpg")
|
|
if err := os.WriteFile(source, []byte("reference"), 0644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
var captured DelegateRequest
|
|
runFn := func(_ context.Context, req DelegateRequest) (DelegateResult, error) {
|
|
captured = req
|
|
staged, err := os.ReadFile(filepath.Join(req.DelegateInputsPath, "reference.jpg"))
|
|
if err != nil {
|
|
return DelegateResult{}, err
|
|
}
|
|
if string(staged) != "reference" {
|
|
return DelegateResult{}, errors.New("staged input content mismatch")
|
|
}
|
|
return DelegateResult{Content: "done"}, nil
|
|
}
|
|
tool := newDelegateTestTool(t, noopAgentLink{}, runFn)
|
|
|
|
ctx := WithToolWorkspace(makeDelegateCtx(t), workspace)
|
|
ctx = WithRunMediaPaths(ctx, []string{source})
|
|
result := tool.Execute(ctx, map[string]any{
|
|
"agent_key": "child-agent",
|
|
"task": "use " + source,
|
|
"mode": "sync",
|
|
})
|
|
|
|
if result == nil || result.IsError {
|
|
t.Fatalf("delegate result = %#v, want success", result)
|
|
}
|
|
if strings.Contains(captured.Task, source) || !strings.Contains(captured.Task, "inputs/reference.jpg") {
|
|
t.Fatalf("Task = %q, want only logical input alias", captured.Task)
|
|
}
|
|
if captured.DelegateInputsPath == "" || captured.DelegateOutputsPath == "" {
|
|
t.Fatalf("runtime artifact roots missing: %#v", captured)
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_DoesNotScrapeAbsolutePathsFromTask(t *testing.T) {
|
|
workspace := t.TempDir()
|
|
source := filepath.Join(workspace, "family-reference.jpg")
|
|
if err := os.WriteFile(source, []byte("reference"), 0644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
var captured DelegateRequest
|
|
var stagedInputCount int
|
|
runFn := func(_ context.Context, req DelegateRequest) (DelegateResult, error) {
|
|
captured = req
|
|
entries, err := os.ReadDir(req.DelegateInputsPath)
|
|
if err != nil {
|
|
return DelegateResult{}, err
|
|
}
|
|
stagedInputCount = len(entries)
|
|
return DelegateResult{Content: "done"}, nil
|
|
}
|
|
tool := newDelegateTestTool(t, noopAgentLink{}, runFn)
|
|
|
|
ctx := WithToolWorkspace(makeDelegateCtx(t), workspace)
|
|
result := tool.Execute(ctx, map[string]any{
|
|
"agent_key": "child-agent",
|
|
"task": "Retry using the exact sample file path: " + source + ".",
|
|
"mode": "sync",
|
|
})
|
|
|
|
if result == nil || result.IsError {
|
|
t.Fatalf("delegate result = %#v, want success", result)
|
|
}
|
|
if stagedInputCount != 0 {
|
|
t.Fatalf("staged input count = %d, want no free-form path scraping", stagedInputCount)
|
|
}
|
|
if !strings.Contains(captured.Task, source) {
|
|
t.Fatalf("Task = %q, want untrusted prose left as prose", captured.Task)
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_RejectsInvalidExplicitInputBeforeDispatch(t *testing.T) {
|
|
workspace := t.TempDir()
|
|
|
|
var called atomic.Bool
|
|
runFn := func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
|
|
called.Store(true)
|
|
return DelegateResult{Content: "done"}, nil
|
|
}
|
|
tool := newDelegateTestTool(t, noopAgentLink{}, runFn)
|
|
|
|
ctx := WithToolWorkspace(makeDelegateCtx(t), workspace)
|
|
result := tool.Execute(ctx, map[string]any{
|
|
"agent_key": "child-agent",
|
|
"task": "Use the requested input",
|
|
"mode": "sync",
|
|
"inputs": []any{"../outside-reference.jpg"},
|
|
})
|
|
|
|
if result == nil || !result.IsError {
|
|
t.Fatalf("delegate result = %#v, want validation error", result)
|
|
}
|
|
if called.Load() {
|
|
t.Fatal("runFn called for invalid explicit input")
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_MissingManagedWorkspaceFailsClosed(t *testing.T) {
|
|
tool := NewDelegateTool(noopAgentLink{}, noopAgentCRUD{}, nil, func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
|
|
return DelegateResult{Content: "unexpected"}, nil
|
|
})
|
|
result := tool.Execute(makeDelegateCtx(t), map[string]any{
|
|
"agent_key": "child-agent",
|
|
"task": "do something",
|
|
"mode": "sync",
|
|
})
|
|
if result == nil || !result.IsError {
|
|
t.Fatalf("delegate result = %#v, want fail-closed setup error", result)
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_PublishesOnlyManifestOutputsAndCleansExchange(t *testing.T) {
|
|
workspace := t.TempDir()
|
|
managedWorkspace := t.TempDir()
|
|
rawMedia := filepath.Join(t.TempDir(), "raw-child-media.png")
|
|
if err := os.WriteFile(rawMedia, []byte("raw"), 0644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
var captured DelegateRequest
|
|
runFn := func(_ context.Context, req DelegateRequest) (DelegateResult, error) {
|
|
captured = req
|
|
if err := os.WriteFile(filepath.Join(req.DelegateOutputsPath, "report.txt"), []byte("published"), 0600); err != nil {
|
|
return DelegateResult{}, err
|
|
}
|
|
return DelegateResult{
|
|
Content: "done",
|
|
Media: []bus.MediaFile{{
|
|
Path: rawMedia,
|
|
MimeType: "image/png",
|
|
Filename: "raw-child-media.png",
|
|
}},
|
|
}, nil
|
|
}
|
|
tool := NewDelegateTool(noopAgentLink{}, noopAgentCRUD{}, nil, runFn)
|
|
tool.SetWorkspace(managedWorkspace)
|
|
|
|
ctx := WithToolWorkspace(makeDelegateCtx(t), workspace)
|
|
result := tool.Execute(ctx, map[string]any{
|
|
"agent_key": "child-agent",
|
|
"task": "create report.txt",
|
|
"mode": "sync",
|
|
})
|
|
if result == nil || result.IsError {
|
|
t.Fatalf("delegate result = %#v, want success", result)
|
|
}
|
|
if len(result.Media) != 1 {
|
|
t.Fatalf("result media = %#v, want one published manifest output", result.Media)
|
|
}
|
|
got := result.Media[0]
|
|
if got.Path == rawMedia || got.Filename != "report.txt" || got.MimeType != "text/plain; charset=utf-8" {
|
|
t.Fatalf("published media = %#v", got)
|
|
}
|
|
if content, err := os.ReadFile(got.Path); err != nil || string(content) != "published" {
|
|
t.Fatalf("published content = %q, %v", content, err)
|
|
}
|
|
if _, err := os.Stat(filepath.Dir(captured.DelegateOutputsPath)); !os.IsNotExist(err) {
|
|
t.Fatalf("successful exchange retained at runtime path: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_AdmissionConstraintsAreTenantScopedAndRootless(t *testing.T) {
|
|
tenantID := uuid.New()
|
|
ctx := store.WithTenantID(context.Background(), tenantID)
|
|
ctx = withSubagentExecution(ctx, TaskScope{
|
|
TenantID: tenantID, RootAgentID: uuid.New(), RootAgentKey: "parent-agent",
|
|
}, "parent-task", 3, nil)
|
|
req := DelegateRequest{
|
|
TenantID: tenantID.String(),
|
|
DelegationID: uuid.NewString(),
|
|
}
|
|
|
|
got := delegateAdmissionConstraints(ctx, req)
|
|
if got.TenantID != tenantID || got.RootAgentID != uuid.Nil {
|
|
t.Fatalf("identity constraints = %#v", got)
|
|
}
|
|
if got.TaskID != req.DelegationID || got.ParentTaskID != "parent-task" {
|
|
t.Fatalf("task constraints = %#v", got)
|
|
}
|
|
if got.Depth != 4 || got.MaxDepth != 0 {
|
|
t.Fatalf("depth constraints = %#v", got)
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_AdmissionDoesNotUseAgentSpawnDepth(t *testing.T) {
|
|
req := DelegateRequest{
|
|
TenantID: store.MasterTenantID.String(),
|
|
DelegationID: uuid.NewString(),
|
|
}
|
|
got := delegateAdmissionConstraints(context.Background(), req)
|
|
if got.Depth != 1 || got.MaxDepth != 0 {
|
|
t.Fatalf("top-level admission depth = %#v, want structural depth 1 without agent cap", got)
|
|
}
|
|
|
|
ctx := WithSubagentConfig(context.Background(), &config.SubagentsConfig{MaxSpawnDepth: 4})
|
|
got = delegateAdmissionConstraints(ctx, req)
|
|
if got.MaxDepth != 0 {
|
|
t.Fatalf("configured agent spawn depth leaked into admission = %#v", got)
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_StagesArtifactsOnlyAfterAdmission(t *testing.T) {
|
|
workspace := t.TempDir()
|
|
managedWorkspace := t.TempDir()
|
|
if err := os.WriteFile(filepath.Join(workspace, "input.txt"), []byte("input"), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
admission := orchestration.NewChildRunAdmission(1, 4)
|
|
defer func() {
|
|
if err := admission.Close(context.Background()); err != nil {
|
|
t.Errorf("close admission: %v", err)
|
|
}
|
|
}()
|
|
firstStarted := make(chan struct{})
|
|
releaseFirst := make(chan struct{})
|
|
secondStarted := make(chan struct{})
|
|
runFn := func(_ context.Context, req DelegateRequest) (DelegateResult, error) {
|
|
if strings.HasPrefix(req.Task, "first") {
|
|
close(firstStarted)
|
|
<-releaseFirst
|
|
} else {
|
|
close(secondStarted)
|
|
}
|
|
return DelegateResult{Content: "done"}, nil
|
|
}
|
|
tool := NewDelegateToolWithAdmission(noopAgentLink{}, noopAgentCRUD{}, nil, runFn, admission)
|
|
tool.SetWorkspace(managedWorkspace)
|
|
ctx := WithToolWorkspace(makeDelegateCtx(t), workspace)
|
|
|
|
first := tool.Execute(ctx, map[string]any{
|
|
"agent_key": "child-agent",
|
|
"task": "first",
|
|
"mode": "async",
|
|
})
|
|
if first == nil || first.IsError {
|
|
t.Fatalf("first delegation = %#v", first)
|
|
}
|
|
select {
|
|
case <-firstStarted:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("first delegation did not start")
|
|
}
|
|
|
|
second := tool.Execute(ctx, map[string]any{
|
|
"agent_key": "child-agent",
|
|
"task": "second",
|
|
"mode": "async",
|
|
"inputs": []any{"input.txt"},
|
|
})
|
|
if second == nil || second.IsError {
|
|
t.Fatalf("second delegation = %#v", second)
|
|
}
|
|
var accepted struct {
|
|
DelegationID string `json:"delegation_id"`
|
|
}
|
|
if err := json.Unmarshal([]byte(second.ForLLM), &accepted); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
secondExchange := filepath.Join(
|
|
managedWorkspace,
|
|
"collaboration",
|
|
"delegations",
|
|
accepted.DelegationID,
|
|
)
|
|
if _, err := os.Stat(secondExchange); !os.IsNotExist(err) {
|
|
t.Fatalf("pending delegation staged before admission: %v", err)
|
|
}
|
|
|
|
close(releaseFirst)
|
|
select {
|
|
case <-secondStarted:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("second delegation did not start after admission")
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_AsyncReleasesAdmissionBeforeAnnouncement(t *testing.T) {
|
|
admission := orchestration.NewChildRunAdmission(1, 4)
|
|
messageBus := bus.New()
|
|
for range 1000 {
|
|
messageBus.PublishInbound(bus.InboundMessage{Content: "fill"})
|
|
}
|
|
|
|
firstStarted := make(chan struct{})
|
|
secondStarted := make(chan struct{})
|
|
var calls atomic.Int32
|
|
runFn := func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
|
|
switch calls.Add(1) {
|
|
case 1:
|
|
close(firstStarted)
|
|
case 2:
|
|
close(secondStarted)
|
|
}
|
|
return DelegateResult{Content: "done"}, nil
|
|
}
|
|
tool := NewDelegateToolWithAdmission(noopAgentLink{}, noopAgentCRUD{}, nil, runFn, admission)
|
|
tool.SetWorkspace(t.TempDir())
|
|
tool.SetMsgBus(messageBus)
|
|
taskStore := newRecordingSubagentTaskStore()
|
|
tool.SetTaskStore(taskStore)
|
|
if tool.announceToParent(DelegateRequest{
|
|
DelegationID: "backpressure-probe",
|
|
ChatID: "chat-1",
|
|
}, "must not block", nil) {
|
|
t.Fatal("full message bus accepted an announcement")
|
|
}
|
|
|
|
ctx := WithToolChatID(makeDelegateCtx(t), "chat-1")
|
|
ctx = WithToolSessionKey(ctx, "delegate-parent-session")
|
|
first := tool.Execute(ctx, map[string]any{
|
|
"agent_key": "child-agent",
|
|
"task": "first",
|
|
"mode": "async",
|
|
})
|
|
if first == nil || first.IsError {
|
|
t.Fatalf("first delegation = %#v, want accepted", first)
|
|
}
|
|
select {
|
|
case <-firstStarted:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("first delegation did not start")
|
|
}
|
|
|
|
second := tool.Execute(ctx, map[string]any{
|
|
"agent_key": "child-agent",
|
|
"task": "second",
|
|
"mode": "async",
|
|
})
|
|
if second == nil || second.IsError {
|
|
t.Fatalf("second delegation = %#v, want accepted", second)
|
|
}
|
|
select {
|
|
case <-secondStarted:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("blocked announce retained the only child-run permit")
|
|
}
|
|
for range 2 {
|
|
select {
|
|
case metadata := <-taskStore.metadata:
|
|
if metadata[asyncCompletionDeliveryKey] != asyncCompletionDeliveryMissed {
|
|
t.Fatalf("announcement metadata = %#v, want undelivered", metadata)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("delegation missed announcement was not recorded after bus saturation")
|
|
}
|
|
}
|
|
|
|
if err := admission.Close(context.Background()); err != nil {
|
|
t.Fatalf("close admission: %v", err)
|
|
}
|
|
tool.Close()
|
|
}
|
|
|
|
func TestDelegateToolCloseContextDrainsAsyncCompletion(t *testing.T) {
|
|
started := make(chan struct{})
|
|
release := make(chan struct{})
|
|
tool := newDelegateTestTool(t, noopAgentLink{}, func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
|
|
close(started)
|
|
<-release
|
|
return DelegateResult{Content: "done"}, nil
|
|
})
|
|
tool.SetTaskStore(newRecordingSubagentTaskStore())
|
|
|
|
result := tool.Execute(makeDelegateCtx(t), map[string]any{
|
|
"agent_key": "child-agent",
|
|
"task": "wait for shutdown drain",
|
|
"mode": "async",
|
|
})
|
|
if result == nil || result.IsError {
|
|
t.Fatalf("delegate result = %#v, want accepted", result)
|
|
}
|
|
select {
|
|
case <-started:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("delegation did not start")
|
|
}
|
|
|
|
timeoutCtx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
|
defer cancel()
|
|
if err := tool.CloseContext(timeoutCtx); err == nil {
|
|
t.Fatal("CloseContext returned before async completion finished")
|
|
}
|
|
|
|
close(release)
|
|
if err := tool.CloseContext(context.Background()); err != nil {
|
|
t.Fatalf("CloseContext after completion: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestDelegationTaskWithInputAliasesDoesNotRewriteProseSubstrings(t *testing.T) {
|
|
task := "Use the data file to update the database metadata."
|
|
inputs := []delegateInput{{
|
|
relativePath: "data",
|
|
}}
|
|
staged := []DelegationArtifact{{Path: "inputs/data"}}
|
|
|
|
got := delegationTaskWithInputAliases(task, inputs, staged)
|
|
if !strings.HasPrefix(got, task) {
|
|
t.Fatalf("task prose was rewritten: %q", got)
|
|
}
|
|
if !strings.Contains(got, "data => inputs/data") {
|
|
t.Fatalf("task is missing the explicit input mapping: %q", got)
|
|
}
|
|
}
|
|
|
|
func TestDelegationTaskWithInputAliasesRewritesOnlyExactPathTokens(t *testing.T) {
|
|
const reference = "/workspace/agent/.uploads/report.pdf"
|
|
task := "Read " + reference + ", but preserve " + reference + ".backup and prefix" + reference
|
|
inputs := []delegateInput{{
|
|
relativePath: ".uploads/report.pdf",
|
|
taskReferences: []string{reference},
|
|
}}
|
|
staged := []DelegationArtifact{{Path: "inputs/report.pdf"}}
|
|
|
|
got := delegationTaskWithInputAliases(task, inputs, staged)
|
|
if !strings.Contains(got, "Read inputs/report.pdf,") {
|
|
t.Fatalf("exact path token was not rewritten: %q", got)
|
|
}
|
|
if !strings.Contains(got, reference+".backup") {
|
|
t.Fatalf("path substring suffix was rewritten: %q", got)
|
|
}
|
|
if !strings.Contains(got, "prefix"+reference) {
|
|
t.Fatalf("path substring prefix was rewritten: %q", got)
|
|
}
|
|
}
|
|
|
|
func TestRedactDelegationArtifactTextHandlesSlashVariants(t *testing.T) {
|
|
exchange := &DelegationArtifactExchange{
|
|
hostRoot: `C:\workspace\collaboration\delegations\123`,
|
|
}
|
|
text := `failed at C:/workspace/collaboration/delegations/123/inputs/file.txt`
|
|
got := redactDelegationArtifactText(text, exchange)
|
|
if strings.Contains(got, "C:/workspace") {
|
|
t.Fatalf("slash-normalized host path was not redacted: %q", got)
|
|
}
|
|
if !strings.Contains(got, "inputs/file.txt") {
|
|
t.Fatalf("logical input path missing after redaction: %q", got)
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_RetainsFailedExchange(t *testing.T) {
|
|
workspace := t.TempDir()
|
|
managedWorkspace := t.TempDir()
|
|
if err := os.WriteFile(filepath.Join(workspace, "input.txt"), []byte("input"), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
var captured DelegateRequest
|
|
tool := NewDelegateTool(noopAgentLink{}, noopAgentCRUD{}, nil, func(_ context.Context, req DelegateRequest) (DelegateResult, error) {
|
|
captured = req
|
|
return DelegateResult{}, errors.New("child run failed")
|
|
})
|
|
tool.SetWorkspace(managedWorkspace)
|
|
t.Cleanup(tool.Close)
|
|
ctx := WithToolWorkspace(makeDelegateCtx(t), workspace)
|
|
result := tool.Execute(ctx, map[string]any{
|
|
"agent_key": "child-agent",
|
|
"task": "fail after staging",
|
|
"mode": "sync",
|
|
"inputs": []any{"input.txt"},
|
|
})
|
|
if result == nil || !result.IsError {
|
|
t.Fatalf("delegate result = %#v, want error", result)
|
|
}
|
|
exchangeRoot := filepath.Join(
|
|
managedWorkspace,
|
|
"collaboration",
|
|
"delegations",
|
|
captured.DelegationID,
|
|
)
|
|
if _, err := os.Stat(filepath.Join(exchangeRoot, "inputs", "input.txt")); err != nil {
|
|
t.Fatalf("failed exchange input not retained: %v", err)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(workspace, ".delegations", captured.DelegationID)); !os.IsNotExist(err) {
|
|
t.Fatalf("failed exchange was published: %v", err)
|
|
}
|
|
}
|
|
|
|
type delegateTraceCaptureStore struct {
|
|
store.TracingStore
|
|
mu sync.Mutex
|
|
spans []store.SpanData
|
|
}
|
|
|
|
func (s *delegateTraceCaptureStore) BatchCreateSpans(_ context.Context, spans []store.SpanData) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.spans = append(s.spans, spans...)
|
|
return nil
|
|
}
|
|
|
|
func (s *delegateTraceCaptureStore) BatchUpdateTraceAggregates(_ context.Context, _ uuid.UUID) error {
|
|
return nil
|
|
}
|
|
|
|
func (s *delegateTraceCaptureStore) RecoverStaleRunningTraces(_ context.Context, _ time.Time) (int64, error) {
|
|
return 0, nil
|
|
}
|
|
|
|
func (s *delegateTraceCaptureStore) DeleteTracesOlderThan(_ context.Context, _ time.Time) (int64, error) {
|
|
return 0, nil
|
|
}
|
|
|
|
func TestDelegateTool_EmitsSanitizedArtifactLifecycleOnDelegateTrace(t *testing.T) {
|
|
workspace := t.TempDir()
|
|
traceStore := &delegateTraceCaptureStore{}
|
|
collector := tracing.NewCollector(traceStore)
|
|
collector.Start()
|
|
traceID := uuid.New()
|
|
|
|
runFn := func(_ context.Context, req DelegateRequest) (DelegateResult, error) {
|
|
if err := os.WriteFile(filepath.Join(req.DelegateOutputsPath, "result.json"), []byte("{}"), 0600); err != nil {
|
|
return DelegateResult{}, err
|
|
}
|
|
return DelegateResult{Content: "done", TraceID: traceID}, nil
|
|
}
|
|
tool := newDelegateTestTool(t, noopAgentLink{}, runFn)
|
|
ctx := tracing.WithCollector(WithToolWorkspace(makeDelegateCtx(t), workspace), collector)
|
|
result := tool.Execute(ctx, map[string]any{
|
|
"agent_key": "child-agent",
|
|
"task": "create result",
|
|
"mode": "sync",
|
|
})
|
|
if result == nil || result.IsError {
|
|
t.Fatalf("delegate result = %#v, want success", result)
|
|
}
|
|
collector.Stop()
|
|
|
|
traceStore.mu.Lock()
|
|
defer traceStore.mu.Unlock()
|
|
if len(traceStore.spans) != 3 {
|
|
t.Fatalf("spans = %#v, want staged, published, cleaned", traceStore.spans)
|
|
}
|
|
wantNames := []string{
|
|
"delegate.artifacts.staged",
|
|
"delegate.artifacts.published",
|
|
"delegate.artifacts.cleaned",
|
|
}
|
|
for i, span := range traceStore.spans {
|
|
if span.TraceID != traceID || span.SpanType != store.SpanTypeEvent ||
|
|
span.Name != wantNames[i] {
|
|
t.Fatalf("lifecycle span %d = %#v, want %s", i, span, wantNames[i])
|
|
}
|
|
metadata := string(span.Metadata)
|
|
if strings.Contains(metadata, workspace) {
|
|
t.Fatalf("lifecycle metadata leaked workspace: %s", metadata)
|
|
}
|
|
}
|
|
for _, index := range []int{1, 2} {
|
|
metadata := string(traceStore.spans[index].Metadata)
|
|
if !strings.Contains(metadata, `"path":"outputs/result.json"`) {
|
|
t.Fatalf("output lifecycle metadata = %s", metadata)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_EmitsSanitizedFailedArtifactLifecycle(t *testing.T) {
|
|
workspace := t.TempDir()
|
|
if err := os.WriteFile(filepath.Join(workspace, "input.txt"), []byte("input"), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
traceStore := &delegateTraceCaptureStore{}
|
|
collector := tracing.NewCollector(traceStore)
|
|
collector.Start()
|
|
traceID := uuid.New()
|
|
|
|
runFn := func(_ context.Context, req DelegateRequest) (DelegateResult, error) {
|
|
if req.OnTraceCreated != nil {
|
|
req.OnTraceCreated(traceID)
|
|
}
|
|
return DelegateResult{TraceID: traceID}, fmt.Errorf(
|
|
"failed while using %s",
|
|
req.DelegateOutputsPath,
|
|
)
|
|
}
|
|
tool := newDelegateTestTool(t, noopAgentLink{}, runFn)
|
|
ctx := tracing.WithCollector(WithToolWorkspace(makeDelegateCtx(t), workspace), collector)
|
|
result := tool.Execute(ctx, map[string]any{
|
|
"agent_key": "child-agent",
|
|
"task": "fail safely",
|
|
"mode": "sync",
|
|
"inputs": []any{"input.txt"},
|
|
})
|
|
if result == nil || !result.IsError {
|
|
t.Fatalf("delegate result = %#v, want failure", result)
|
|
}
|
|
if strings.Contains(result.ForLLM, workspace) {
|
|
t.Fatalf("delegate error leaked workspace: %s", result.ForLLM)
|
|
}
|
|
collector.Stop()
|
|
|
|
traceStore.mu.Lock()
|
|
defer traceStore.mu.Unlock()
|
|
if len(traceStore.spans) != 2 {
|
|
t.Fatalf("spans = %#v, want staged and failed", traceStore.spans)
|
|
}
|
|
wantNames := []string{
|
|
"delegate.artifacts.staged",
|
|
"delegate.artifacts.failed",
|
|
}
|
|
for i, span := range traceStore.spans {
|
|
if span.TraceID != traceID || span.Name != wantNames[i] {
|
|
t.Fatalf("failure lifecycle span %d = %#v", i, span)
|
|
}
|
|
metadata := string(span.Metadata)
|
|
if strings.Contains(metadata, workspace) ||
|
|
!strings.Contains(metadata, `"path":"inputs/input.txt"`) {
|
|
t.Fatalf("failure lifecycle metadata = %s", metadata)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_EmitsCancelledArtifactLifecycle(t *testing.T) {
|
|
traceStore := &delegateTraceCaptureStore{}
|
|
collector := tracing.NewCollector(traceStore)
|
|
collector.Start()
|
|
traceID := uuid.New()
|
|
|
|
runFn := func(_ context.Context, req DelegateRequest) (DelegateResult, error) {
|
|
if req.OnTraceCreated != nil {
|
|
req.OnTraceCreated(traceID)
|
|
}
|
|
return DelegateResult{TraceID: traceID}, context.Canceled
|
|
}
|
|
tool := newDelegateTestTool(t, noopAgentLink{}, runFn)
|
|
ctx := tracing.WithCollector(makeDelegateCtx(t), collector)
|
|
result := tool.Execute(ctx, map[string]any{
|
|
"agent_key": "child-agent",
|
|
"task": "cancel safely",
|
|
"mode": "sync",
|
|
})
|
|
if result == nil || !result.IsError {
|
|
t.Fatalf("delegate result = %#v, want cancellation", result)
|
|
}
|
|
collector.Stop()
|
|
|
|
traceStore.mu.Lock()
|
|
defer traceStore.mu.Unlock()
|
|
if len(traceStore.spans) != 2 ||
|
|
traceStore.spans[0].Name != "delegate.artifacts.staged" ||
|
|
traceStore.spans[1].Name != "delegate.artifacts.cancelled" {
|
|
t.Fatalf("cancellation lifecycle spans = %#v", traceStore.spans)
|
|
}
|
|
for _, span := range traceStore.spans {
|
|
if span.TraceID != traceID {
|
|
t.Fatalf("cancellation trace ID = %s, want %s", span.TraceID, traceID)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_SweeperDeletesOnlyRegisteredExpiredExchange(t *testing.T) {
|
|
tenantWorkspace := t.TempDir()
|
|
delegationID := uuid.New()
|
|
exchangeRoot := filepath.Join(tenantWorkspace, "collaboration", "delegations", delegationID.String())
|
|
if err := os.MkdirAll(exchangeRoot, 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(exchangeRoot, "retained.txt"), []byte("retained"), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
outside := t.TempDir()
|
|
sentinel := filepath.Join(outside, "sentinel.txt")
|
|
if err := os.WriteFile(sentinel, []byte("keep"), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.Symlink(outside, filepath.Join(exchangeRoot, "hostile-link")); err != nil {
|
|
t.Logf("symlink cleanup coverage unavailable: %v", err)
|
|
}
|
|
tool := NewDelegateTool(noopAgentLink{}, noopAgentCRUD{}, nil, func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
|
|
return DelegateResult{}, nil
|
|
})
|
|
item := retainedDelegationArtifact{
|
|
tenantWorkspace: tenantWorkspace,
|
|
tenantID: store.MasterTenantID,
|
|
delegationID: delegationID,
|
|
retainUntil: time.Now().Add(-time.Minute),
|
|
}
|
|
tool.retained[retainedDelegationArtifactKey(tenantWorkspace, delegationID)] = item
|
|
tool.sweepRetainedDelegationExchanges(time.Now())
|
|
|
|
if _, err := os.Stat(exchangeRoot); !os.IsNotExist(err) {
|
|
t.Fatalf("expired exchange still exists: %v", err)
|
|
}
|
|
if len(tool.retained) != 0 {
|
|
t.Fatalf("retained registry = %#v, want empty", tool.retained)
|
|
}
|
|
if got, err := os.ReadFile(sentinel); err != nil || string(got) != "keep" {
|
|
t.Fatalf("secure cleanup followed hostile link: %q, %v", got, err)
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_RecoversRetainedExchangeAfterRestart(t *testing.T) {
|
|
tenantWorkspace := t.TempDir()
|
|
callerWorkspace := filepath.Join(tenantWorkspace, "agents", "caller")
|
|
if err := os.MkdirAll(callerWorkspace, 0750); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
callerRoot, err := OpenDelegationArtifactRoot(callerWorkspace)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
delegationID := uuid.New()
|
|
exchange, err := NewDelegationArtifactExchange(
|
|
tenantWorkspace,
|
|
store.MasterTenantID,
|
|
delegationID,
|
|
DelegationArtifactLimits{},
|
|
time.Minute,
|
|
)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
first := NewDelegateTool(noopAgentLink{}, noopAgentCRUD{}, nil, func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
|
|
return DelegateResult{}, nil
|
|
})
|
|
first.SetWorkspace(tenantWorkspace)
|
|
job := &delegateArtifactJob{
|
|
req: DelegateRequest{DelegationID: delegationID.String()},
|
|
callerRoot: callerRoot,
|
|
callerWorkspace: callerWorkspace,
|
|
tenantWorkspace: tenantWorkspace,
|
|
tenantID: store.MasterTenantID,
|
|
delegationID: delegationID,
|
|
}
|
|
job.callerLocation = first.resolveDelegationCallerLocation(job)
|
|
exchange.RetainFailure(time.Now().Add(-2*time.Minute), "test_failure")
|
|
if err := first.registerRetainedDelegationExchange(exchange, job, artifactLifecycleFailed); err != nil {
|
|
t.Fatalf("register retained exchange: %v", err)
|
|
}
|
|
stateBytes, err := os.ReadFile(filepath.Join(
|
|
tenantWorkspace,
|
|
"collaboration",
|
|
"delegations",
|
|
delegationID.String(),
|
|
delegationArtifactLifecycleFile,
|
|
))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Contains(string(stateBytes), tenantWorkspace) ||
|
|
strings.Contains(string(stateBytes), callerWorkspace) {
|
|
t.Fatalf("lifecycle state persisted a host path: %s", stateBytes)
|
|
}
|
|
if err := exchange.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := callerRoot.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
first.Close()
|
|
|
|
restarted := NewDelegateTool(noopAgentLink{}, noopAgentCRUD{}, nil, func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
|
|
return DelegateResult{}, nil
|
|
})
|
|
restarted.SetWorkspace(tenantWorkspace)
|
|
if len(restarted.retained) != 1 {
|
|
t.Fatalf("recovered retained registry = %#v, want one", restarted.retained)
|
|
}
|
|
restarted.sweepRetainedDelegationExchanges(time.Now())
|
|
restarted.Close()
|
|
|
|
exchangeRoot := filepath.Join(
|
|
tenantWorkspace,
|
|
"collaboration",
|
|
"delegations",
|
|
delegationID.String(),
|
|
)
|
|
if _, err := os.Stat(exchangeRoot); !os.IsNotExist(err) {
|
|
t.Fatalf("restart janitor left expired exchange: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_RecoversStaleArtifactLifecycleStates(t *testing.T) {
|
|
testCases := []struct {
|
|
name string
|
|
status delegationArtifactLifecycleStatus
|
|
}{
|
|
{name: "staging", status: artifactLifecycleStaging},
|
|
{name: "running", status: artifactLifecycleRunning},
|
|
{name: "publishing", status: artifactLifecyclePublishing},
|
|
}
|
|
|
|
for _, testCase := range testCases {
|
|
t.Run(testCase.name, func(t *testing.T) {
|
|
tenantWorkspace := t.TempDir()
|
|
callerWorkspace := filepath.Join(tenantWorkspace, "agents", "caller")
|
|
if err := os.MkdirAll(callerWorkspace, 0750); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
callerRoot, err := OpenDelegationArtifactRoot(callerWorkspace)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
delegationID := uuid.New()
|
|
exchange, err := NewDelegationArtifactExchange(
|
|
tenantWorkspace,
|
|
store.MasterTenantID,
|
|
delegationID,
|
|
DelegationArtifactLimits{},
|
|
time.Minute,
|
|
)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
first := NewDelegateTool(
|
|
noopAgentLink{},
|
|
noopAgentCRUD{},
|
|
nil,
|
|
func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
|
|
return DelegateResult{}, nil
|
|
},
|
|
)
|
|
first.workspace = tenantWorkspace
|
|
job := &delegateArtifactJob{
|
|
req: DelegateRequest{DelegationID: delegationID.String()},
|
|
callerRoot: callerRoot,
|
|
callerWorkspace: callerWorkspace,
|
|
tenantWorkspace: tenantWorkspace,
|
|
tenantID: store.MasterTenantID,
|
|
delegationID: delegationID,
|
|
}
|
|
job.callerLocation = first.resolveDelegationCallerLocation(job)
|
|
if err := first.updateActiveDelegationLifecycle(exchange, job); err != nil {
|
|
t.Fatalf("update staging lifecycle: %v", err)
|
|
}
|
|
|
|
publicationTempPath := ""
|
|
if testCase.status == artifactLifecycleRunning ||
|
|
testCase.status == artifactLifecyclePublishing {
|
|
if err := first.markDelegationRunning(exchange, job, time.Now()); err != nil {
|
|
t.Fatalf("mark running: %v", err)
|
|
}
|
|
}
|
|
if testCase.status == artifactLifecyclePublishing {
|
|
publicationTempPath = filepath.ToSlash(filepath.Join(
|
|
".delegations",
|
|
".tmp-"+delegationID.String()+"-"+uuid.NewString(),
|
|
))
|
|
if err := first.markDelegationPublishing(
|
|
exchange,
|
|
job,
|
|
publicationTempPath,
|
|
time.Now(),
|
|
); err != nil {
|
|
t.Fatalf("mark publishing: %v", err)
|
|
}
|
|
if err := os.MkdirAll(
|
|
filepath.Join(callerWorkspace, filepath.FromSlash(publicationTempPath)),
|
|
0700,
|
|
); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
finalSentinel := filepath.Join(
|
|
callerWorkspace,
|
|
".delegations",
|
|
delegationID.String(),
|
|
"keep.txt",
|
|
)
|
|
if err := os.MkdirAll(filepath.Dir(finalSentinel), 0750); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(finalSentinel, []byte("keep"), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := exchange.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := callerRoot.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
first.Close()
|
|
|
|
restarted := NewDelegateTool(
|
|
noopAgentLink{},
|
|
noopAgentCRUD{},
|
|
nil,
|
|
func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
|
|
return DelegateResult{}, nil
|
|
},
|
|
)
|
|
recoveredAfter := time.Now().UTC()
|
|
restarted.SetWorkspace(tenantWorkspace)
|
|
defer restarted.Close()
|
|
if len(restarted.retained) != 1 {
|
|
t.Fatalf("recovered registry = %#v, want one", restarted.retained)
|
|
}
|
|
|
|
lifecyclePath := filepath.Join(
|
|
tenantWorkspace,
|
|
"collaboration",
|
|
"delegations",
|
|
delegationID.String(),
|
|
delegationArtifactLifecycleFile,
|
|
)
|
|
stateBytes, err := os.ReadFile(lifecyclePath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var state delegationArtifactLifecycleState
|
|
if err := json.Unmarshal(stateBytes, &state); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if state.Status != artifactLifecycleFailed ||
|
|
state.FailedAt == nil ||
|
|
state.ReasonCode != "artifact_recovered_stale" {
|
|
t.Fatalf("recovered state = %#v", state)
|
|
}
|
|
earliestExpiry := recoveredAfter.Add(delegationArtifactFailureTTL)
|
|
if state.RetainUntil.Before(earliestExpiry) ||
|
|
state.RetainUntil.After(time.Now().UTC().Add(delegationArtifactFailureTTL)) {
|
|
t.Fatalf("retain until = %v, want recovery time + %v", state.RetainUntil, delegationArtifactFailureTTL)
|
|
}
|
|
|
|
restarted.sweepRetainedDelegationExchanges(state.RetainUntil.Add(-time.Nanosecond))
|
|
if _, err := os.Stat(lifecyclePath); err != nil {
|
|
t.Fatalf("retained exchange removed before TTL: %v", err)
|
|
}
|
|
restarted.sweepRetainedDelegationExchanges(state.RetainUntil)
|
|
if _, err := os.Stat(filepath.Dir(lifecyclePath)); !os.IsNotExist(err) {
|
|
t.Fatalf("expired exchange remains: %v", err)
|
|
}
|
|
if publicationTempPath != "" {
|
|
if _, err := os.Stat(filepath.Join(
|
|
callerWorkspace,
|
|
filepath.FromSlash(publicationTempPath),
|
|
)); !os.IsNotExist(err) {
|
|
t.Fatalf("recorded publication temp remains: %v", err)
|
|
}
|
|
}
|
|
if got, err := os.ReadFile(finalSentinel); err != nil || string(got) != "keep" {
|
|
t.Fatalf("final UUID publication directory was modified: %q, %v", got, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_PeriodicArtifactMaintenancePreservesActiveExchange(t *testing.T) {
|
|
tenantWorkspace := t.TempDir()
|
|
callerWorkspace := filepath.Join(tenantWorkspace, "agents", "caller")
|
|
if err := os.MkdirAll(callerWorkspace, 0o750); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
callerRoot, err := OpenDelegationArtifactRoot(callerWorkspace)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer callerRoot.Close()
|
|
|
|
delegationID := uuid.New()
|
|
runStarted := make(chan struct{})
|
|
resumeRun := make(chan struct{})
|
|
var tool *DelegateTool
|
|
tool = NewDelegateTool(
|
|
noopAgentLink{},
|
|
noopAgentCRUD{},
|
|
nil,
|
|
func(_ context.Context, req DelegateRequest) (DelegateResult, error) {
|
|
close(runStarted)
|
|
<-resumeRun
|
|
if !tool.delegationArtifactExchangeActive(tenantWorkspace, delegationID) {
|
|
return DelegateResult{}, errors.New("delegation artifact exchange became inactive during run")
|
|
}
|
|
if err := os.WriteFile(
|
|
filepath.Join(req.DelegateOutputsPath, "result.txt"),
|
|
[]byte("active result"),
|
|
0o600,
|
|
); err != nil {
|
|
return DelegateResult{}, err
|
|
}
|
|
return DelegateResult{Content: "done"}, nil
|
|
},
|
|
)
|
|
tool.workspace = tenantWorkspace
|
|
defer tool.Close()
|
|
|
|
job := &delegateArtifactJob{
|
|
req: DelegateRequest{DelegationID: delegationID.String()},
|
|
callerRoot: callerRoot,
|
|
callerWorkspace: callerWorkspace,
|
|
tenantWorkspace: tenantWorkspace,
|
|
tenantID: store.MasterTenantID,
|
|
delegationID: delegationID,
|
|
}
|
|
job.callerLocation = tool.resolveDelegationCallerLocation(job)
|
|
|
|
runDone := make(chan error, 1)
|
|
go func() {
|
|
_, err := tool.runArtifactExchange(context.Background(), job)
|
|
runDone <- err
|
|
}()
|
|
select {
|
|
case <-runStarted:
|
|
case <-time.After(2 * time.Second):
|
|
close(resumeRun)
|
|
t.Fatal("delegated run did not start")
|
|
}
|
|
tool.addRetainedDelegationArtifact(retainedDelegationArtifact{
|
|
tenantWorkspace: tenantWorkspace,
|
|
tenantID: store.MasterTenantID,
|
|
delegationID: uuid.New(),
|
|
retainUntil: time.Now().Add(time.Hour),
|
|
})
|
|
tool.maintainDelegationArtifacts(time.Now())
|
|
|
|
lifecyclePath := filepath.Join(
|
|
tenantWorkspace,
|
|
"collaboration",
|
|
"delegations",
|
|
delegationID.String(),
|
|
delegationArtifactLifecycleFile,
|
|
)
|
|
stateBytes, stateErr := os.ReadFile(lifecyclePath)
|
|
var state delegationArtifactLifecycleState
|
|
if stateErr == nil {
|
|
stateErr = json.Unmarshal(stateBytes, &state)
|
|
}
|
|
close(resumeRun)
|
|
runErr := <-runDone
|
|
if stateErr != nil {
|
|
t.Fatal(stateErr)
|
|
}
|
|
if state.Status != artifactLifecycleRunning {
|
|
t.Fatalf("active lifecycle status = %q, want %q", state.Status, artifactLifecycleRunning)
|
|
}
|
|
if runErr != nil {
|
|
t.Fatalf("active exchange run after maintenance: %v", runErr)
|
|
}
|
|
if tool.delegationArtifactExchangeActive(tenantWorkspace, delegationID) {
|
|
t.Fatal("completed delegation artifact exchange remained active")
|
|
}
|
|
publishedOutput := filepath.Join(
|
|
callerWorkspace,
|
|
".delegations",
|
|
delegationID.String(),
|
|
"outputs",
|
|
"result.txt",
|
|
)
|
|
if got, err := os.ReadFile(publishedOutput); err != nil || string(got) != "active result" {
|
|
t.Fatalf("published output = %q, %v", got, err)
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_PeriodicArtifactMaintenanceRecoversUnregisteredExchange(t *testing.T) {
|
|
tenantWorkspace := t.TempDir()
|
|
callerWorkspace := filepath.Join(tenantWorkspace, "agents", "caller")
|
|
if err := os.MkdirAll(callerWorkspace, 0o750); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
callerRoot, err := OpenDelegationArtifactRoot(callerWorkspace)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer callerRoot.Close()
|
|
|
|
delegationID := uuid.New()
|
|
exchange, err := NewDelegationArtifactExchange(
|
|
tenantWorkspace,
|
|
store.MasterTenantID,
|
|
delegationID,
|
|
DelegationArtifactLimits{},
|
|
time.Minute,
|
|
)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer exchange.Close()
|
|
|
|
tool := NewDelegateTool(
|
|
noopAgentLink{},
|
|
noopAgentCRUD{},
|
|
nil,
|
|
func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
|
|
return DelegateResult{}, nil
|
|
},
|
|
)
|
|
tool.workspace = tenantWorkspace
|
|
defer tool.Close()
|
|
|
|
job := &delegateArtifactJob{
|
|
req: DelegateRequest{DelegationID: delegationID.String()},
|
|
callerRoot: callerRoot,
|
|
callerWorkspace: callerWorkspace,
|
|
tenantWorkspace: tenantWorkspace,
|
|
tenantID: store.MasterTenantID,
|
|
delegationID: delegationID,
|
|
}
|
|
job.callerLocation = tool.resolveDelegationCallerLocation(job)
|
|
if err := tool.updateActiveDelegationLifecycle(exchange, job); err != nil {
|
|
t.Fatalf("update active lifecycle: %v", err)
|
|
}
|
|
if err := tool.markDelegationRunning(exchange, job, time.Now()); err != nil {
|
|
t.Fatalf("mark running: %v", err)
|
|
}
|
|
|
|
tool.maintainDelegationArtifacts(time.Now())
|
|
|
|
state, err := readDelegationArtifactLifecycleState(exchange.root, delegationID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if state.Status != artifactLifecycleFailed || state.ReasonCode != "artifact_recovered_stale" {
|
|
t.Fatalf("unregistered lifecycle = %#v, want recovered stale failure", state)
|
|
}
|
|
if _, ok := tool.retained[retainedDelegationArtifactKey(tenantWorkspace, delegationID)]; !ok {
|
|
t.Fatalf("unregistered exchange was not added to retained registry: %#v", tool.retained)
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_RecoveryPromotesDurablePublishingState(t *testing.T) {
|
|
tenantWorkspace := t.TempDir()
|
|
callerWorkspace := filepath.Join(tenantWorkspace, "agents", "caller")
|
|
if err := os.MkdirAll(callerWorkspace, 0o750); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
callerRoot, err := OpenDelegationArtifactRoot(callerWorkspace)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
delegationID := uuid.New()
|
|
exchange, err := NewDelegationArtifactExchange(
|
|
tenantWorkspace,
|
|
store.MasterTenantID,
|
|
delegationID,
|
|
DelegationArtifactLimits{},
|
|
time.Minute,
|
|
)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
first := NewDelegateTool(
|
|
noopAgentLink{},
|
|
noopAgentCRUD{},
|
|
nil,
|
|
func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
|
|
return DelegateResult{}, nil
|
|
},
|
|
)
|
|
first.workspace = tenantWorkspace
|
|
job := &delegateArtifactJob{
|
|
req: DelegateRequest{DelegationID: delegationID.String()},
|
|
callerRoot: callerRoot,
|
|
callerWorkspace: callerWorkspace,
|
|
tenantWorkspace: tenantWorkspace,
|
|
tenantID: store.MasterTenantID,
|
|
delegationID: delegationID,
|
|
}
|
|
job.callerLocation = first.resolveDelegationCallerLocation(job)
|
|
if err := first.updateActiveDelegationLifecycle(exchange, job); err != nil {
|
|
t.Fatalf("update lifecycle: %v", err)
|
|
}
|
|
if err := first.markDelegationRunning(exchange, job, time.Now()); err != nil {
|
|
t.Fatalf("mark running: %v", err)
|
|
}
|
|
if err := os.WriteFile(
|
|
filepath.Join(exchange.OutputsHostPath(), "result.txt"),
|
|
[]byte("durable result"),
|
|
0o600,
|
|
); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
publication, err := exchange.publishWithPreparation(
|
|
context.Background(),
|
|
callerRoot,
|
|
time.Now(),
|
|
func(tempPath string) error {
|
|
return first.markDelegationPublishing(exchange, job, tempPath, time.Now())
|
|
},
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("publish before simulated crash: %v", err)
|
|
}
|
|
if err := exchange.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := callerRoot.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
first.Close()
|
|
|
|
restarted := NewDelegateTool(
|
|
noopAgentLink{},
|
|
noopAgentCRUD{},
|
|
nil,
|
|
func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
|
|
return DelegateResult{}, nil
|
|
},
|
|
)
|
|
restarted.SetWorkspace(tenantWorkspace)
|
|
defer restarted.Close()
|
|
|
|
item, ok := restarted.retained[retainedDelegationArtifactKey(tenantWorkspace, delegationID)]
|
|
if !ok || !item.publicationDurable || !item.retainUntil.IsZero() {
|
|
t.Fatalf("recovered publication = %#v, want durable immediate cleanup", item)
|
|
}
|
|
lifecyclePath := filepath.Join(
|
|
tenantWorkspace,
|
|
"collaboration",
|
|
"delegations",
|
|
delegationID.String(),
|
|
delegationArtifactLifecycleFile,
|
|
)
|
|
stateBytes, err := os.ReadFile(lifecyclePath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var state delegationArtifactLifecycleState
|
|
if err := json.Unmarshal(stateBytes, &state); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if state.Status != artifactLifecyclePublished || state.ReasonCode != "artifact_published" {
|
|
t.Fatalf("recovered lifecycle = %#v, want published", state)
|
|
}
|
|
|
|
restarted.sweepRetainedDelegationExchanges(time.Now())
|
|
if _, err := os.Stat(filepath.Dir(lifecyclePath)); !os.IsNotExist(err) {
|
|
t.Fatalf("published exchange remains after cleanup: %v", err)
|
|
}
|
|
publishedOutput := filepath.Join(
|
|
callerWorkspace,
|
|
filepath.FromSlash(publication.RootPath),
|
|
"outputs",
|
|
"result.txt",
|
|
)
|
|
if got, err := os.ReadFile(publishedOutput); err != nil || string(got) != "durable result" {
|
|
t.Fatalf("durable publication changed during recovery: %q, %v", got, err)
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_RecoveryCleansInvalidAndExpiresCorruptExchanges(t *testing.T) {
|
|
tenantWorkspace := t.TempDir()
|
|
delegationsRoot := filepath.Join(tenantWorkspace, "collaboration", "delegations")
|
|
invalidRoot := filepath.Join(delegationsRoot, "not-a-delegation")
|
|
if err := os.MkdirAll(invalidRoot, 0750); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
corruptID := uuid.New()
|
|
corruptRoot := filepath.Join(delegationsRoot, corruptID.String())
|
|
if err := os.MkdirAll(corruptRoot, 0750); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(
|
|
filepath.Join(corruptRoot, delegationArtifactLifecycleFile),
|
|
[]byte("{not-json"),
|
|
0600,
|
|
); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
tool := NewDelegateTool(
|
|
noopAgentLink{},
|
|
noopAgentCRUD{},
|
|
nil,
|
|
func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
|
|
return DelegateResult{}, nil
|
|
},
|
|
)
|
|
tool.SetWorkspace(tenantWorkspace)
|
|
defer tool.Close()
|
|
|
|
if _, err := os.Stat(invalidRoot); !os.IsNotExist(err) {
|
|
t.Fatalf("invalid exchange entry remains after recovery: %v", err)
|
|
}
|
|
item, ok := tool.retained[retainedDelegationArtifactKey(tenantWorkspace, corruptID)]
|
|
if !ok {
|
|
t.Fatalf("corrupt exchange was not registered for bounded retention: %#v", tool.retained)
|
|
}
|
|
tool.sweepRetainedDelegationExchanges(item.retainUntil.Add(-time.Nanosecond))
|
|
if _, err := os.Stat(corruptRoot); err != nil {
|
|
t.Fatalf("corrupt exchange removed before retention elapsed: %v", err)
|
|
}
|
|
tool.sweepRetainedDelegationExchanges(item.retainUntil)
|
|
if _, err := os.Stat(corruptRoot); !os.IsNotExist(err) {
|
|
t.Fatalf("corrupt exchange remains after retention elapsed: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_RecoversPublicationCleanupAfterRestart(t *testing.T) {
|
|
tenantWorkspace := t.TempDir()
|
|
callerWorkspace := filepath.Join(tenantWorkspace, "agents", "caller")
|
|
if err := os.MkdirAll(callerWorkspace, 0750); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
callerRoot, err := OpenDelegationArtifactRoot(callerWorkspace)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
delegationID := uuid.New()
|
|
exchange, err := NewDelegationArtifactExchange(
|
|
tenantWorkspace,
|
|
store.MasterTenantID,
|
|
delegationID,
|
|
DelegationArtifactLimits{},
|
|
time.Minute,
|
|
)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
first := NewDelegateTool(
|
|
noopAgentLink{},
|
|
noopAgentCRUD{},
|
|
nil,
|
|
func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
|
|
return DelegateResult{}, nil
|
|
},
|
|
)
|
|
first.workspace = tenantWorkspace
|
|
job := &delegateArtifactJob{
|
|
req: DelegateRequest{DelegationID: delegationID.String()},
|
|
callerRoot: callerRoot,
|
|
callerWorkspace: callerWorkspace,
|
|
tenantWorkspace: tenantWorkspace,
|
|
tenantID: store.MasterTenantID,
|
|
delegationID: delegationID,
|
|
}
|
|
job.callerLocation = first.resolveDelegationCallerLocation(job)
|
|
if err := first.updateActiveDelegationLifecycle(exchange, job); err != nil {
|
|
t.Fatalf("update staging lifecycle: %v", err)
|
|
}
|
|
if err := first.markDelegationRunning(exchange, job, time.Now()); err != nil {
|
|
t.Fatalf("mark running: %v", err)
|
|
}
|
|
if err := os.WriteFile(
|
|
filepath.Join(exchange.OutputsHostPath(), "result.txt"),
|
|
[]byte("durable"),
|
|
0600,
|
|
); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var publicationTempPath string
|
|
publication, err := exchange.publishWithPreparation(
|
|
context.Background(),
|
|
callerRoot,
|
|
time.Now(),
|
|
func(tempPath string) error {
|
|
publicationTempPath = tempPath
|
|
return first.markDelegationPublishing(exchange, job, tempPath, time.Now())
|
|
},
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("publish before simulated restart: %v", err)
|
|
}
|
|
if err := first.markDelegationPublished(exchange, job, publication.Manifest.PublishedAt); err != nil {
|
|
t.Fatalf("mark published: %v", err)
|
|
}
|
|
if publicationTempPath == "" {
|
|
t.Fatal("publication temp path was not recorded")
|
|
}
|
|
if err := exchange.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := callerRoot.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
first.Close()
|
|
|
|
restarted := NewDelegateTool(
|
|
noopAgentLink{},
|
|
noopAgentCRUD{},
|
|
nil,
|
|
func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
|
|
return DelegateResult{}, nil
|
|
},
|
|
)
|
|
restarted.SetWorkspace(tenantWorkspace)
|
|
if len(restarted.retained) != 1 {
|
|
t.Fatalf("recovered cleanup registry = %#v, want one", restarted.retained)
|
|
}
|
|
restarted.sweepRetainedDelegationExchanges(time.Now())
|
|
restarted.Close()
|
|
|
|
exchangeRoot := filepath.Join(
|
|
tenantWorkspace,
|
|
"collaboration",
|
|
"delegations",
|
|
delegationID.String(),
|
|
)
|
|
if _, err := os.Stat(exchangeRoot); !os.IsNotExist(err) {
|
|
t.Fatalf("restart janitor left publication exchange: %v", err)
|
|
}
|
|
publicationRoot := filepath.Join(
|
|
callerWorkspace,
|
|
".delegations",
|
|
delegationID.String(),
|
|
)
|
|
published, err := os.ReadFile(filepath.Join(publicationRoot, "outputs", "result.txt"))
|
|
if err != nil || string(published) != "durable" {
|
|
t.Fatalf("restart janitor damaged durable publication: %q, %v", published, err)
|
|
}
|
|
}
|
|
|
|
func TestDelegateTool_RetriesPublishedExchangeCleanup(t *testing.T) {
|
|
managedWorkspace := t.TempDir()
|
|
callerWorkspace := t.TempDir()
|
|
traceStore := &delegateTraceCaptureStore{}
|
|
collector := tracing.NewCollector(traceStore)
|
|
collector.Start()
|
|
traceID := uuid.New()
|
|
var captured DelegateRequest
|
|
tool := NewDelegateTool(
|
|
noopAgentLink{},
|
|
noopAgentCRUD{},
|
|
nil,
|
|
func(_ context.Context, req DelegateRequest) (DelegateResult, error) {
|
|
captured = req
|
|
if err := os.WriteFile(
|
|
filepath.Join(req.DelegateOutputsPath, "result.txt"),
|
|
[]byte("durable"),
|
|
0600,
|
|
); err != nil {
|
|
return DelegateResult{}, err
|
|
}
|
|
return DelegateResult{Content: "done", TraceID: traceID}, nil
|
|
},
|
|
)
|
|
tool.SetWorkspace(managedWorkspace)
|
|
defer tool.Close()
|
|
|
|
removeCalls := 0
|
|
tool.removeExchange = func(tenantWorkspace string, delegationID uuid.UUID) error {
|
|
removeCalls++
|
|
if removeCalls == 1 {
|
|
return errors.New("injected cleanup failure")
|
|
}
|
|
return tool.removeDelegationExchange(tenantWorkspace, delegationID)
|
|
}
|
|
ctx := tracing.WithCollector(
|
|
WithToolWorkspace(makeDelegateCtx(t), callerWorkspace),
|
|
collector,
|
|
)
|
|
result := tool.Execute(ctx, map[string]any{
|
|
"agent_key": "child-agent",
|
|
"task": "publish output",
|
|
"mode": "sync",
|
|
})
|
|
if result == nil || result.IsError {
|
|
t.Fatalf("delegate result = %#v, want success", result)
|
|
}
|
|
delegationID := uuid.MustParse(captured.DelegationID)
|
|
exchangeRoot := filepath.Join(
|
|
managedWorkspace,
|
|
"collaboration",
|
|
"delegations",
|
|
delegationID.String(),
|
|
)
|
|
if _, err := os.Stat(exchangeRoot); err != nil {
|
|
t.Fatalf("failed cleanup did not retain exchange for retry: %v", err)
|
|
}
|
|
tool.retainedMu.Lock()
|
|
retainedCount := len(tool.retained)
|
|
sweeperStarted := tool.sweeperStarted
|
|
tool.retainedMu.Unlock()
|
|
if retainedCount != 1 || !sweeperStarted {
|
|
t.Fatalf("automatic cleanup registry = %d, sweeper started = %v", retainedCount, sweeperStarted)
|
|
}
|
|
|
|
tool.sweepRetainedDelegationExchanges(time.Now())
|
|
if removeCalls != 2 {
|
|
t.Fatalf("cleanup calls = %d, want immediate attempt plus retry", removeCalls)
|
|
}
|
|
if _, err := os.Stat(exchangeRoot); !os.IsNotExist(err) {
|
|
t.Fatalf("cleanup retry left exchange: %v", err)
|
|
}
|
|
published, err := os.ReadFile(filepath.Join(
|
|
callerWorkspace,
|
|
".delegations",
|
|
delegationID.String(),
|
|
"outputs",
|
|
"result.txt",
|
|
))
|
|
if err != nil || string(published) != "durable" {
|
|
t.Fatalf("cleanup retry damaged durable output: %q, %v", published, err)
|
|
}
|
|
collector.Stop()
|
|
traceStore.mu.Lock()
|
|
defer traceStore.mu.Unlock()
|
|
if len(traceStore.spans) != 3 ||
|
|
traceStore.spans[2].TraceID != traceID ||
|
|
traceStore.spans[2].Name != "delegate.artifacts.cleaned" {
|
|
t.Fatalf("cleanup retry spans = %#v, want cleaned on original trace", traceStore.spans)
|
|
}
|
|
}
|