mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-13 02:14:39 +00:00
fix(collaboration): preserve active artifact exchanges
This commit is contained in:
1 parent
9d5f9ce857
commit
b6fb91b163
3 files changed
+234
-11
No files matched your search
@@ -86,6 +86,9 @@ type DelegateTool struct {
|
||||
sweeperStop chan struct{}
|
||||
sweeperDone chan struct{}
|
||||
|
||||
activeArtifactMu sync.RWMutex
|
||||
activeArtifacts map[string]uint32
|
||||
|
||||
completionMu sync.Mutex
|
||||
completionClosed bool
|
||||
completionWG sync.WaitGroup
|
||||
@@ -190,15 +193,16 @@ func NewDelegateToolWithAdmission(
|
||||
admission = orchestration.NewChildRunAdmission(32, 128)
|
||||
}
|
||||
return &DelegateTool{
|
||||
links: links,
|
||||
agents: agents,
|
||||
eventBus: eb,
|
||||
runFn: runFn,
|
||||
admission: admission,
|
||||
retained: make(map[string]retainedDelegationArtifact),
|
||||
sweeperStop: make(chan struct{}),
|
||||
sweeperDone: make(chan struct{}),
|
||||
closeDone: make(chan struct{}),
|
||||
links: links,
|
||||
agents: agents,
|
||||
eventBus: eb,
|
||||
runFn: runFn,
|
||||
admission: admission,
|
||||
retained: make(map[string]retainedDelegationArtifact),
|
||||
sweeperStop: make(chan struct{}),
|
||||
sweeperDone: make(chan struct{}),
|
||||
activeArtifacts: make(map[string]uint32),
|
||||
closeDone: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -717,6 +721,9 @@ func collectDelegateInputs(callerWorkspace string, explicitInputs, currentMedia
|
||||
}
|
||||
|
||||
func (t *DelegateTool) runArtifactExchange(ctx context.Context, job *delegateArtifactJob) (dr DelegateResult, returnErr error) {
|
||||
t.beginDelegationArtifactExchange(job.tenantWorkspace, job.delegationID)
|
||||
defer t.endDelegationArtifactExchange(job.tenantWorkspace, job.delegationID)
|
||||
|
||||
exchange, err := NewDelegationArtifactExchange(
|
||||
job.tenantWorkspace,
|
||||
job.tenantID,
|
||||
|
||||
@@ -1800,6 +1800,178 @@ func TestDelegateTool_RecoversStaleArtifactLifecycleStates(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelegateTool_PeriodicArtifactMaintenancePreservesActiveExchange(t *testing.T) {
|
||||
tenantWorkspace := t.TempDir()
|
||||
callerWorkspace := filepath.Join(tenantWorkspace, "agents", "caller")
|
||||
if err := os.MkdirAll(callerWorkspace, 0o750); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
callerRoot, err := OpenDelegationArtifactRoot(callerWorkspace)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer callerRoot.Close()
|
||||
|
||||
delegationID := uuid.New()
|
||||
runStarted := make(chan struct{})
|
||||
resumeRun := make(chan struct{})
|
||||
var tool *DelegateTool
|
||||
tool = NewDelegateTool(
|
||||
noopAgentLink{},
|
||||
noopAgentCRUD{},
|
||||
nil,
|
||||
func(_ context.Context, req DelegateRequest) (DelegateResult, error) {
|
||||
close(runStarted)
|
||||
<-resumeRun
|
||||
if !tool.delegationArtifactExchangeActive(tenantWorkspace, delegationID) {
|
||||
return DelegateResult{}, errors.New("delegation artifact exchange became inactive during run")
|
||||
}
|
||||
if err := os.WriteFile(
|
||||
filepath.Join(req.DelegateOutputsPath, "result.txt"),
|
||||
[]byte("active result"),
|
||||
0o600,
|
||||
); err != nil {
|
||||
return DelegateResult{}, err
|
||||
}
|
||||
return DelegateResult{Content: "done"}, nil
|
||||
},
|
||||
)
|
||||
tool.workspace = tenantWorkspace
|
||||
defer tool.Close()
|
||||
|
||||
job := &delegateArtifactJob{
|
||||
req: DelegateRequest{DelegationID: delegationID.String()},
|
||||
callerRoot: callerRoot,
|
||||
callerWorkspace: callerWorkspace,
|
||||
tenantWorkspace: tenantWorkspace,
|
||||
tenantID: store.MasterTenantID,
|
||||
delegationID: delegationID,
|
||||
}
|
||||
job.callerLocation = tool.resolveDelegationCallerLocation(job)
|
||||
|
||||
runDone := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := tool.runArtifactExchange(context.Background(), job)
|
||||
runDone <- err
|
||||
}()
|
||||
select {
|
||||
case <-runStarted:
|
||||
case <-time.After(2 * time.Second):
|
||||
close(resumeRun)
|
||||
t.Fatal("delegated run did not start")
|
||||
}
|
||||
tool.addRetainedDelegationArtifact(retainedDelegationArtifact{
|
||||
tenantWorkspace: tenantWorkspace,
|
||||
tenantID: store.MasterTenantID,
|
||||
delegationID: uuid.New(),
|
||||
retainUntil: time.Now().Add(time.Hour),
|
||||
})
|
||||
tool.maintainDelegationArtifacts(time.Now())
|
||||
|
||||
lifecyclePath := filepath.Join(
|
||||
tenantWorkspace,
|
||||
"collaboration",
|
||||
"delegations",
|
||||
delegationID.String(),
|
||||
delegationArtifactLifecycleFile,
|
||||
)
|
||||
stateBytes, stateErr := os.ReadFile(lifecyclePath)
|
||||
var state delegationArtifactLifecycleState
|
||||
if stateErr == nil {
|
||||
stateErr = json.Unmarshal(stateBytes, &state)
|
||||
}
|
||||
close(resumeRun)
|
||||
runErr := <-runDone
|
||||
if stateErr != nil {
|
||||
t.Fatal(stateErr)
|
||||
}
|
||||
if state.Status != artifactLifecycleRunning {
|
||||
t.Fatalf("active lifecycle status = %q, want %q", state.Status, artifactLifecycleRunning)
|
||||
}
|
||||
if runErr != nil {
|
||||
t.Fatalf("active exchange run after maintenance: %v", runErr)
|
||||
}
|
||||
if tool.delegationArtifactExchangeActive(tenantWorkspace, delegationID) {
|
||||
t.Fatal("completed delegation artifact exchange remained active")
|
||||
}
|
||||
publishedOutput := filepath.Join(
|
||||
callerWorkspace,
|
||||
".delegations",
|
||||
delegationID.String(),
|
||||
"outputs",
|
||||
"result.txt",
|
||||
)
|
||||
if got, err := os.ReadFile(publishedOutput); err != nil || string(got) != "active result" {
|
||||
t.Fatalf("published output = %q, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelegateTool_PeriodicArtifactMaintenanceRecoversUnregisteredExchange(t *testing.T) {
|
||||
tenantWorkspace := t.TempDir()
|
||||
callerWorkspace := filepath.Join(tenantWorkspace, "agents", "caller")
|
||||
if err := os.MkdirAll(callerWorkspace, 0o750); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
callerRoot, err := OpenDelegationArtifactRoot(callerWorkspace)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer callerRoot.Close()
|
||||
|
||||
delegationID := uuid.New()
|
||||
exchange, err := NewDelegationArtifactExchange(
|
||||
tenantWorkspace,
|
||||
store.MasterTenantID,
|
||||
delegationID,
|
||||
DelegationArtifactLimits{},
|
||||
time.Minute,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer exchange.Close()
|
||||
|
||||
tool := NewDelegateTool(
|
||||
noopAgentLink{},
|
||||
noopAgentCRUD{},
|
||||
nil,
|
||||
func(_ context.Context, _ DelegateRequest) (DelegateResult, error) {
|
||||
return DelegateResult{}, nil
|
||||
},
|
||||
)
|
||||
tool.workspace = tenantWorkspace
|
||||
defer tool.Close()
|
||||
|
||||
job := &delegateArtifactJob{
|
||||
req: DelegateRequest{DelegationID: delegationID.String()},
|
||||
callerRoot: callerRoot,
|
||||
callerWorkspace: callerWorkspace,
|
||||
tenantWorkspace: tenantWorkspace,
|
||||
tenantID: store.MasterTenantID,
|
||||
delegationID: delegationID,
|
||||
}
|
||||
job.callerLocation = tool.resolveDelegationCallerLocation(job)
|
||||
if err := tool.updateActiveDelegationLifecycle(exchange, job); err != nil {
|
||||
t.Fatalf("update active lifecycle: %v", err)
|
||||
}
|
||||
if err := tool.markDelegationRunning(exchange, job, time.Now()); err != nil {
|
||||
t.Fatalf("mark running: %v", err)
|
||||
}
|
||||
|
||||
tool.maintainDelegationArtifacts(time.Now())
|
||||
|
||||
state, err := readDelegationArtifactLifecycleState(exchange.root, delegationID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state.Status != artifactLifecycleFailed || state.ReasonCode != "artifact_recovered_stale" {
|
||||
t.Fatalf("unregistered lifecycle = %#v, want recovered stale failure", state)
|
||||
}
|
||||
if _, ok := tool.retained[retainedDelegationArtifactKey(tenantWorkspace, delegationID)]; !ok {
|
||||
t.Fatalf("unregistered exchange was not added to retained registry: %#v", tool.retained)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelegateTool_RecoveryPromotesDurablePublishingState(t *testing.T) {
|
||||
tenantWorkspace := t.TempDir()
|
||||
callerWorkspace := filepath.Join(tenantWorkspace, "agents", "caller")
|
||||
|
||||
@@ -263,6 +263,43 @@ func retainedDelegationArtifactKey(tenantWorkspace string, delegationID uuid.UUI
|
||||
return tenantWorkspace + "\x00" + delegationID.String()
|
||||
}
|
||||
|
||||
func (t *DelegateTool) beginDelegationArtifactExchange(
|
||||
tenantWorkspace string,
|
||||
delegationID uuid.UUID,
|
||||
) {
|
||||
key := retainedDelegationArtifactKey(tenantWorkspace, delegationID)
|
||||
t.activeArtifactMu.Lock()
|
||||
defer t.activeArtifactMu.Unlock()
|
||||
if t.activeArtifacts == nil {
|
||||
t.activeArtifacts = make(map[string]uint32)
|
||||
}
|
||||
t.activeArtifacts[key]++
|
||||
}
|
||||
|
||||
func (t *DelegateTool) endDelegationArtifactExchange(
|
||||
tenantWorkspace string,
|
||||
delegationID uuid.UUID,
|
||||
) {
|
||||
key := retainedDelegationArtifactKey(tenantWorkspace, delegationID)
|
||||
t.activeArtifactMu.Lock()
|
||||
defer t.activeArtifactMu.Unlock()
|
||||
if t.activeArtifacts[key] <= 1 {
|
||||
delete(t.activeArtifacts, key)
|
||||
return
|
||||
}
|
||||
t.activeArtifacts[key]--
|
||||
}
|
||||
|
||||
func (t *DelegateTool) delegationArtifactExchangeActive(
|
||||
tenantWorkspace string,
|
||||
delegationID uuid.UUID,
|
||||
) bool {
|
||||
key := retainedDelegationArtifactKey(tenantWorkspace, delegationID)
|
||||
t.activeArtifactMu.RLock()
|
||||
defer t.activeArtifactMu.RUnlock()
|
||||
return t.activeArtifacts[key] > 0
|
||||
}
|
||||
|
||||
func (t *DelegateTool) runDelegationArtifactSweeper() {
|
||||
ticker := time.NewTicker(delegationArtifactSweepInterval)
|
||||
defer func() {
|
||||
@@ -272,14 +309,18 @@ func (t *DelegateTool) runDelegationArtifactSweeper() {
|
||||
for {
|
||||
select {
|
||||
case now := <-ticker.C:
|
||||
t.recoverRetainedDelegationExchanges()
|
||||
t.sweepRetainedDelegationExchanges(now)
|
||||
t.maintainDelegationArtifacts(now)
|
||||
case <-t.sweeperStop:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (t *DelegateTool) maintainDelegationArtifacts(now time.Time) {
|
||||
t.recoverRetainedDelegationExchanges()
|
||||
t.sweepRetainedDelegationExchanges(now)
|
||||
}
|
||||
|
||||
func (t *DelegateTool) recoverRetainedDelegationExchanges() {
|
||||
for _, tenantWorkspace := range t.delegationTenantWorkspaces() {
|
||||
if err := t.recoverTenantDelegationExchanges(tenantWorkspace); err != nil {
|
||||
@@ -350,6 +391,9 @@ func (t *DelegateTool) recoverTenantDelegationExchanges(tenantWorkspace string)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if t.delegationArtifactExchangeActive(tenantWorkspace, delegationID) {
|
||||
continue
|
||||
}
|
||||
exchangeRoot, err := delegationsRoot.openSubroot(name)
|
||||
if err != nil {
|
||||
slog.Warn("delegate.artifact_exchange_recovery_open_failed",
|
||||
|
||||
Reference in new issue
Block a user