fix(collaboration): preserve active artifact exchanges

This commit is contained in:
ntduc committed 2026-07-31 08:39:50 +07:00
1 parent 9d5f9ce857
commit b6fb91b163
3 files changed
+234 -11

No files matched your search

+16 -9
View File
@@ -86,6 +86,9 @@ type DelegateTool struct {
sweeperStop chan struct{}
sweeperDone chan struct{}
activeArtifactMu sync.RWMutex
activeArtifacts map[string]uint32
completionMu sync.Mutex
completionClosed bool
completionWG sync.WaitGroup
@@ -190,15 +193,16 @@ func NewDelegateToolWithAdmission(
admission = orchestration.NewChildRunAdmission(32, 128)
}
return &DelegateTool{
links: links,
agents: agents,
eventBus: eb,
runFn: runFn,
admission: admission,
retained: make(map[string]retainedDelegationArtifact),
sweeperStop: make(chan struct{}),
sweeperDone: make(chan struct{}),
closeDone: make(chan struct{}),
links: links,
agents: agents,
eventBus: eb,
runFn: runFn,
admission: admission,
retained: make(map[string]retainedDelegationArtifact),
sweeperStop: make(chan struct{}),
sweeperDone: make(chan struct{}),
activeArtifacts: make(map[string]uint32),
closeDone: make(chan struct{}),
}
}
@@ -717,6 +721,9 @@ func collectDelegateInputs(callerWorkspace string, explicitInputs, currentMedia
}
func (t *DelegateTool) runArtifactExchange(ctx context.Context, job *delegateArtifactJob) (dr DelegateResult, returnErr error) {
t.beginDelegationArtifactExchange(job.tenantWorkspace, job.delegationID)
defer t.endDelegationArtifactExchange(job.tenantWorkspace, job.delegationID)
exchange, err := NewDelegationArtifactExchange(
job.tenantWorkspace,
job.tenantID,
+172
View File
@@ -1800,6 +1800,178 @@ func TestDelegateTool_RecoversStaleArtifactLifecycleStates(t *testing.T) {
}
}
func TestDelegateTool_PeriodicArtifactMaintenancePreservesActiveExchange(t *testing.T) {
tenantWorkspace := t.TempDir()
callerWorkspace := filepath.Join(tenantWorkspace, "agents", "caller")
if err := os.MkdirAll(callerWorkspace, 0o750); err != nil {
t.Fatal(err)
}
callerRoot, err := OpenDelegationArtifactRoot(callerWorkspace)
if err != nil {
t.Fatal(err)
}
defer callerRoot.Close()
delegationID := uuid.New()
runStarted := make(chan struct{})
resumeRun := make(chan struct{})
var tool *DelegateTool
tool = NewDelegateTool(
noopAgentLink{},
noopAgentCRUD{},
nil,
func(_ context.Context, req DelegateRequest) (DelegateResult, error) {
close(runStarted)
<-resumeRun
if !tool.delegationArtifactExchangeActive(tenantWorkspace, delegationID) {
return DelegateResult{}, errors.New("delegation artifact exchange became inactive during run")
}
if err := os.WriteFile(
filepath.Join(req.DelegateOutputsPath, "result.txt"),
[]byte("active result"),
0o600,
); err != nil {
return DelegateResult{}, err
}
return DelegateResult{Content: "done"}, nil
},
)
tool.workspace = tenantWorkspace
defer tool.Close()
job := &delegateArtifactJob{
req: DelegateRequest{DelegationID: delegationID.String()},
callerRoot: callerRoot,
callerWorkspace: callerWorkspace,
tenantWorkspace: tenantWorkspace,
tenantID: store.MasterTenantID,
delegationID: delegationID,
}
job.callerLocation = tool.resolveDelegationCallerLocation(job)
runDone := make(chan error, 1)
go func() {
_, err := tool.runArtifactExchange(context.Background(), job)
runDone <- err
}()
select {
case <-runStarted:
case <-time.After(2 * time.Second):
close(resumeRun)
t.Fatal("delegated run did not start")
}
tool.addRetainedDelegationArtifact(retainedDelegationArtifact{
tenantWorkspace: tenantWorkspace,
tenantID: store.MasterTenantID,
delegationID: uuid.New(),
retainUntil: time.Now().Add(time.Hour),
})
tool.maintainDelegationArtifacts(time.Now())
lifecyclePath := filepath.Join(
tenantWorkspace,
"collaboration",
"delegations",
delegationID.String(),
delegationArtifactLifecycleFile,
)
stateBytes, stateErr := os.ReadFile(lifecyclePath)
var state delegationArtifactLifecycleState
if stateErr == nil {
stateErr = json.Unmarshal(stateBytes, &state)
}
close(resumeRun)
runErr := <-runDone
if stateErr != nil {
t.Fatal(stateErr)
}
if state.Status != artifactLifecycleRunning {
t.Fatalf("active lifecycle status = %q, want %q", state.Status, artifactLifecycleRunning)
}
if runErr != nil {
t.Fatalf("active exchange run after maintenance: %v", runErr)
}
if tool.delegationArtifactExchangeActive(tenantWorkspace, delegationID) {
t.Fatal("completed delegation artifact exchange remained active")
}
publishedOutput := filepath.Join(
callerWorkspace,
".delegations",
delegationID.String(),
"outputs",
"result.txt",
)
if got, err := os.ReadFile(publishedOutput); err != nil || string(got) != "active result" {
t.Fatalf("published output = %q, %v", got, err)
}
}
func TestDelegateTool_PeriodicArtifactMaintenanceRecoversUnregisteredExchange(t *testing.T) {
tenantWorkspace := t.TempDir()
callerWorkspace := filepath.Join(tenantWorkspace, "agents", "caller")
if err := os.MkdirAll(callerWorkspace, 0o750); err != nil {
t.Fatal(err)
}
callerRoot, err := OpenDelegationArtifactRoot(callerWorkspace)
if err != nil {
t.Fatal(err)
}
defer callerRoot.Close()
delegationID := uuid.New()
exchange, err := NewDelegationArtifactExchange(
tenantWorkspace,
store.MasterTenantID,
delegationID,
DelegationArtifactLimits{},
time.Minute,
)
if err != nil {
t.Fatal(err)
}
defer exchange.Close()
tool := NewDelegateTool(
noopAgentLink{},
noopAgentCRUD{},
nil,
func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
return DelegateResult{}, nil
},
)
tool.workspace = tenantWorkspace
defer tool.Close()
job := &delegateArtifactJob{
req: DelegateRequest{DelegationID: delegationID.String()},
callerRoot: callerRoot,
callerWorkspace: callerWorkspace,
tenantWorkspace: tenantWorkspace,
tenantID: store.MasterTenantID,
delegationID: delegationID,
}
job.callerLocation = tool.resolveDelegationCallerLocation(job)
if err := tool.updateActiveDelegationLifecycle(exchange, job); err != nil {
t.Fatalf("update active lifecycle: %v", err)
}
if err := tool.markDelegationRunning(exchange, job, time.Now()); err != nil {
t.Fatalf("mark running: %v", err)
}
tool.maintainDelegationArtifacts(time.Now())
state, err := readDelegationArtifactLifecycleState(exchange.root, delegationID)
if err != nil {
t.Fatal(err)
}
if state.Status != artifactLifecycleFailed || state.ReasonCode != "artifact_recovered_stale" {
t.Fatalf("unregistered lifecycle = %#v, want recovered stale failure", state)
}
if _, ok := tool.retained[retainedDelegationArtifactKey(tenantWorkspace, delegationID)]; !ok {
t.Fatalf("unregistered exchange was not added to retained registry: %#v", tool.retained)
}
}
func TestDelegateTool_RecoveryPromotesDurablePublishingState(t *testing.T) {
tenantWorkspace := t.TempDir()
callerWorkspace := filepath.Join(tenantWorkspace, "agents", "caller")
+46 -2
View File
@@ -263,6 +263,43 @@ func retainedDelegationArtifactKey(tenantWorkspace string, delegationID uuid.UUI
return tenantWorkspace + "\x00" + delegationID.String()
}
func (t *DelegateTool) beginDelegationArtifactExchange(
tenantWorkspace string,
delegationID uuid.UUID,
) {
key := retainedDelegationArtifactKey(tenantWorkspace, delegationID)
t.activeArtifactMu.Lock()
defer t.activeArtifactMu.Unlock()
if t.activeArtifacts == nil {
t.activeArtifacts = make(map[string]uint32)
}
t.activeArtifacts[key]++
}
func (t *DelegateTool) endDelegationArtifactExchange(
tenantWorkspace string,
delegationID uuid.UUID,
) {
key := retainedDelegationArtifactKey(tenantWorkspace, delegationID)
t.activeArtifactMu.Lock()
defer t.activeArtifactMu.Unlock()
if t.activeArtifacts[key] <= 1 {
delete(t.activeArtifacts, key)
return
}
t.activeArtifacts[key]--
}
func (t *DelegateTool) delegationArtifactExchangeActive(
tenantWorkspace string,
delegationID uuid.UUID,
) bool {
key := retainedDelegationArtifactKey(tenantWorkspace, delegationID)
t.activeArtifactMu.RLock()
defer t.activeArtifactMu.RUnlock()
return t.activeArtifacts[key] > 0
}
func (t *DelegateTool) runDelegationArtifactSweeper() {
ticker := time.NewTicker(delegationArtifactSweepInterval)
defer func() {
@@ -272,14 +309,18 @@ func (t *DelegateTool) runDelegationArtifactSweeper() {
for {
select {
case now := <-ticker.C:
t.recoverRetainedDelegationExchanges()
t.sweepRetainedDelegationExchanges(now)
t.maintainDelegationArtifacts(now)
case <-t.sweeperStop:
return
}
}
}
func (t *DelegateTool) maintainDelegationArtifacts(now time.Time) {
t.recoverRetainedDelegationExchanges()
t.sweepRetainedDelegationExchanges(now)
}
func (t *DelegateTool) recoverRetainedDelegationExchanges() {
for _, tenantWorkspace := range t.delegationTenantWorkspaces() {
if err := t.recoverTenantDelegationExchanges(tenantWorkspace); err != nil {
@@ -350,6 +391,9 @@ func (t *DelegateTool) recoverTenantDelegationExchanges(tenantWorkspace string)
}
continue
}
if t.delegationArtifactExchangeActive(tenantWorkspace, delegationID) {
continue
}
exchangeRoot, err := delegationsRoot.openSubroot(name)
if err != nil {
slog.Warn("delegate.artifact_exchange_recovery_open_failed",