mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-17 22:14:14 +00:00
fix(collaboration): harden delegated task isolation (#1486)
* feat(collaboration): isolate delegated artifacts and child runs Isolate delegated inputs and outputs behind secure artifact exchange lifecycles. Scope Agent Link tasks by tenant and root agent, and enforce delegation spawn-tree boundaries. Add process-wide child-run admission and preserve logical media paths across native, MCP, and sandbox execution. * fix(collaboration): harden delegated task isolation Enforce tenant and root-agent task scope across migrations and stores. Add exactly-once async completion delivery, delegated sandbox boundaries, and confined artifact and media recovery across runtime surfaces. * fix(collaboration): recover interrupted async tasks * fix(collaboration): normalize persisted child-run status --------- Co-authored-by: ntduc <ntduc@cpp.ai.vn>
This commit is contained in:
Duc Nguyenandntduc authored and GitHub committed
1 parent
496b7ffce6
commit
bb7712a9ff
186 files changed
+18858
-1233
No files matched your search
+103
-56
@@ -16,10 +16,15 @@ import (
|
||||
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
|
||||
)
|
||||
|
||||
// runTask executes the subagent in a goroutine.
|
||||
func (sm *SubagentManager) runTask(ctx context.Context, task *SubagentTask, callback AsyncCallback) {
|
||||
iterations := sm.executeTask(ctx, task)
|
||||
|
||||
// announceTask publishes terminal state after child-run execution capacity has
|
||||
// been released. Slow bus consumers and callbacks must never hold admission.
|
||||
func (sm *SubagentManager) announceTask(
|
||||
ctx context.Context,
|
||||
task *SubagentTask,
|
||||
callback AsyncCallback,
|
||||
iterations int,
|
||||
terminalPersisted bool,
|
||||
) {
|
||||
// Announce result to parent via bus (matching TS subagent-announce.ts pattern).
|
||||
// The announce goes through the parent agent's session so the agent can
|
||||
// reformulate the result for the user.
|
||||
@@ -27,15 +32,19 @@ func (sm *SubagentManager) runTask(ctx context.Context, task *SubagentTask, call
|
||||
elapsed := time.Since(time.UnixMilli(task.CreatedAt))
|
||||
|
||||
item := AnnounceQueueItem{
|
||||
SubagentID: task.ID,
|
||||
Label: task.Label,
|
||||
Status: task.Status,
|
||||
Result: task.Result,
|
||||
Media: task.Media,
|
||||
Runtime: elapsed,
|
||||
Iterations: iterations,
|
||||
InputTokens: task.TotalInputTokens,
|
||||
OutputTokens: task.TotalOutputTokens,
|
||||
SubagentID: task.ID,
|
||||
CompletionID: task.dbID,
|
||||
DurablyPersisted: terminalPersisted,
|
||||
ParentTaskID: task.ParentTaskID,
|
||||
Depth: task.Depth,
|
||||
Label: task.Label,
|
||||
Status: task.Status,
|
||||
Result: task.Result,
|
||||
Media: task.Media,
|
||||
Runtime: elapsed,
|
||||
Iterations: iterations,
|
||||
InputTokens: task.TotalInputTokens,
|
||||
OutputTokens: task.TotalOutputTokens,
|
||||
}
|
||||
meta := AnnounceMetadata{
|
||||
OriginChannel: task.OriginChannel,
|
||||
@@ -47,34 +56,40 @@ func (sm *SubagentManager) runTask(ctx context.Context, task *SubagentTask, call
|
||||
OriginRole: task.OriginRole,
|
||||
OriginSessionKey: task.OriginSessionKey,
|
||||
OriginTenantID: task.OriginTenantID,
|
||||
ParentAgent: task.ParentID,
|
||||
RootAgentID: task.RootAgentID,
|
||||
ParentAgent: task.RootAgentKey,
|
||||
OriginTraceID: task.OriginTraceID.String(),
|
||||
OriginRootSpanID: task.OriginRootSpanID.String(),
|
||||
}
|
||||
|
||||
if sm.announceQueue != nil {
|
||||
// Use batched announce queue (matching TS debounce pattern)
|
||||
sessionKey := fmt.Sprintf("announce:%s:%s", task.ParentID, task.OriginChatID)
|
||||
sessionKey := subagentAnnounceBatchKey(task)
|
||||
sm.announceQueue.Enqueue(sessionKey, item, meta)
|
||||
} else {
|
||||
// Direct publish (no batching)
|
||||
roster := sm.RosterForParent(task.ParentID)
|
||||
roster := sm.RosterForParent(TaskScope{
|
||||
TenantID: task.OriginTenantID, RootAgentID: task.RootAgentID, RootAgentKey: task.RootAgentKey,
|
||||
})
|
||||
announceContent := FormatBatchedAnnounce([]AnnounceQueueItem{item}, roster)
|
||||
|
||||
announceMeta := map[string]string{
|
||||
MetaOriginChannel: task.OriginChannel,
|
||||
MetaOriginPeerKind: task.OriginPeerKind,
|
||||
MetaParentAgent: task.ParentID,
|
||||
"subagent_id": task.ID,
|
||||
MetaSubagentLabel: task.Label,
|
||||
MetaSubagentStatus: task.Status,
|
||||
MetaSubagentResult: task.Result,
|
||||
MetaSubagentRuntime: fmt.Sprintf("%d", elapsed.Milliseconds()),
|
||||
MetaSubagentIterations: fmt.Sprintf("%d", iterations),
|
||||
MetaSubagentInputToks: fmt.Sprintf("%d", task.TotalInputTokens),
|
||||
MetaSubagentOutputToks: fmt.Sprintf("%d", task.TotalOutputTokens),
|
||||
MetaOriginTraceID: task.OriginTraceID.String(),
|
||||
MetaOriginRootSpanID: task.OriginRootSpanID.String(),
|
||||
MetaOriginChannel: task.OriginChannel,
|
||||
MetaOriginPeerKind: task.OriginPeerKind,
|
||||
MetaParentAgent: task.RootAgentKey,
|
||||
MetaSubagentRootAgentID: task.RootAgentID.String(),
|
||||
"subagent_id": task.ID,
|
||||
MetaSubagentLabel: task.Label,
|
||||
MetaSubagentStatus: task.Status,
|
||||
MetaSubagentResult: task.Result,
|
||||
MetaSubagentRuntime: fmt.Sprintf("%d", elapsed.Milliseconds()),
|
||||
MetaSubagentIterations: fmt.Sprintf("%d", iterations),
|
||||
MetaSubagentInputToks: fmt.Sprintf("%d", task.TotalInputTokens),
|
||||
MetaSubagentOutputToks: fmt.Sprintf("%d", task.TotalOutputTokens),
|
||||
MetaSubagentParentTask: task.ParentTaskID,
|
||||
MetaSubagentDepth: fmt.Sprintf("%d", task.Depth),
|
||||
MetaOriginTraceID: task.OriginTraceID.String(),
|
||||
MetaOriginRootSpanID: task.OriginRootSpanID.String(),
|
||||
}
|
||||
if task.OriginLocalKey != "" {
|
||||
announceMeta[MetaOriginLocalKey] = task.OriginLocalKey
|
||||
@@ -91,7 +106,7 @@ func (sm *SubagentManager) runTask(ctx context.Context, task *SubagentTask, call
|
||||
if task.OriginUserID != "" {
|
||||
announceMeta[MetaOriginUserID] = task.OriginUserID
|
||||
}
|
||||
sm.msgBus.PublishInbound(bus.InboundMessage{
|
||||
delivered := PublishAsyncCompletion(ctx, sm.msgBus, bus.InboundMessage{
|
||||
Channel: "system",
|
||||
SenderID: fmt.Sprintf("subagent:%s", task.ID),
|
||||
ChatID: task.OriginChatID,
|
||||
@@ -101,6 +116,24 @@ func (sm *SubagentManager) runTask(ctx context.Context, task *SubagentTask, call
|
||||
Metadata: announceMeta,
|
||||
Media: task.Media,
|
||||
})
|
||||
if terminalPersisted {
|
||||
sm.UpdateAnnouncementStatus(ctx, task.RootAgentID, task.dbID, delivered)
|
||||
} else {
|
||||
slog.Error("subagent.announce_without_durable_terminal",
|
||||
"task_id", task.ID,
|
||||
"completion_id", task.dbID,
|
||||
"root_agent_id", task.RootAgentID,
|
||||
"delivered", delivered,
|
||||
)
|
||||
}
|
||||
if !delivered {
|
||||
slog.Warn("subagent.announce_deferred_to_ledger",
|
||||
"task_id", task.ID,
|
||||
"completion_id", task.dbID,
|
||||
"root_agent_id", task.RootAgentID,
|
||||
"reason", "inbound_bus_full",
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -112,6 +145,23 @@ func (sm *SubagentManager) runTask(ctx context.Context, task *SubagentTask, call
|
||||
}
|
||||
}
|
||||
|
||||
func subagentAnnounceBatchKey(task *SubagentTask) string {
|
||||
return strings.Join([]string{
|
||||
"announce",
|
||||
task.OriginTenantID.String(),
|
||||
task.RootAgentID.String(),
|
||||
task.RootAgentKey,
|
||||
task.OriginSessionKey,
|
||||
task.OriginChannel,
|
||||
task.OriginChatID,
|
||||
task.OriginPeerKind,
|
||||
task.OriginLocalKey,
|
||||
task.OriginUserID,
|
||||
task.OriginSenderID,
|
||||
task.OriginRole,
|
||||
}, "\x00")
|
||||
}
|
||||
|
||||
// executeTask runs the LLM tool loop for a subagent. Returns iteration count.
|
||||
func (sm *SubagentManager) executeTask(ctx context.Context, task *SubagentTask) int {
|
||||
// Tracing: generate a root span ID for this subagent execution.
|
||||
@@ -120,19 +170,15 @@ func (sm *SubagentManager) executeTask(ctx context.Context, task *SubagentTask)
|
||||
subRootSpanID := store.GenNewID()
|
||||
taskStart := time.Now().UTC()
|
||||
|
||||
// Use a detached context for tracing so spans are emitted even if parent ctx is cancelled.
|
||||
// We copy tracing values but remove the cancellation chain.
|
||||
traceCtx := context.Background()
|
||||
if collector := tracing.CollectorFromContext(ctx); collector != nil {
|
||||
traceCtx = tracing.WithCollector(traceCtx, collector)
|
||||
traceCtx = tracing.WithTraceID(traceCtx, tracing.TraceIDFromContext(ctx))
|
||||
// Keep original parent_span_id (parent agent's root span) for the subagent root span.
|
||||
traceCtx = tracing.WithParentSpanID(traceCtx, tracing.ParentSpanIDFromContext(ctx))
|
||||
}
|
||||
// Detach cancellation while preserving the delegation redactor and the
|
||||
// original trace parent. Rebuilding from Background would silently drop the
|
||||
// artifact confidentiality boundary on nested synchronous work.
|
||||
traceCtx := context.WithoutCancel(ctx)
|
||||
|
||||
// subCtx overrides parent_span_id so child spans nest under subRootSpanID.
|
||||
// traceCtx retains the original parent_span_id for the root subagent span.
|
||||
subTraceCtx := tracing.WithParentSpanID(traceCtx, subRootSpanID)
|
||||
toolCtx := tracing.WithParentSpanID(ctx, subRootSpanID)
|
||||
|
||||
var model string
|
||||
var finalContent string
|
||||
@@ -150,10 +196,6 @@ func (sm *SubagentManager) executeTask(ctx context.Context, task *SubagentTask)
|
||||
"trace_id", tracing.TraceIDFromContext(traceCtx),
|
||||
"status", task.Status, "iterations", iteration)
|
||||
|
||||
// Schedule auto-archive
|
||||
if task.spawnConfig.ArchiveAfterMinutes > 0 {
|
||||
go sm.scheduleArchive(task.ID, time.Duration(task.spawnConfig.ArchiveAfterMinutes)*time.Minute)
|
||||
}
|
||||
}()
|
||||
|
||||
if ctx.Err() != nil {
|
||||
@@ -166,6 +208,7 @@ func (sm *SubagentManager) executeTask(ctx context.Context, task *SubagentTask)
|
||||
|
||||
// Build tools for subagent (no spawn/subagent tools to prevent recursion)
|
||||
toolsReg := sm.createTools()
|
||||
toolsReg.Register(NewSpawnTool(sm, task.RootAgentKey, task.Depth))
|
||||
sm.applyDenyList(toolsReg, task.Depth, task.spawnConfig)
|
||||
|
||||
// Determine model (cascading priority):
|
||||
@@ -201,11 +244,15 @@ func (sm *SubagentManager) executeTask(ctx context.Context, task *SubagentTask)
|
||||
|
||||
// Build subagent system prompt (matching TS buildSubagentSystemPrompt pattern).
|
||||
workspace := ToolWorkspaceFromCtx(ctx)
|
||||
systemPrompt := sm.buildSubagentSystemPrompt(task, task.spawnConfig, workspace)
|
||||
promptWorkspace := workspace
|
||||
if IsDelegationArtifactRun(ctx) {
|
||||
promptWorkspace = "outputs/"
|
||||
}
|
||||
systemPrompt := tracing.RedactText(ctx, sm.buildSubagentSystemPrompt(task, task.spawnConfig, promptWorkspace))
|
||||
|
||||
messages := []providers.Message{
|
||||
{Role: "system", Content: systemPrompt},
|
||||
{Role: "user", Content: task.Task},
|
||||
{Role: "user", Content: tracing.RedactText(ctx, task.Task)},
|
||||
}
|
||||
|
||||
// Run LLM iteration loop (similar to agent loop but simplified)
|
||||
@@ -273,10 +320,9 @@ func (sm *SubagentManager) executeTask(ctx context.Context, task *SubagentTask)
|
||||
if err != nil {
|
||||
sm.mu.Lock()
|
||||
task.Status = TaskStatusFailed
|
||||
task.Result = fmt.Sprintf("LLM error at iteration %d: %v", iteration, err)
|
||||
task.Result = tracing.RedactText(ctx, fmt.Sprintf("LLM error at iteration %d: %v", iteration, err))
|
||||
sm.mu.Unlock()
|
||||
slog.Warn("subagent LLM error", "id", task.ID, "iteration", iteration, "error", err)
|
||||
go sm.persistStatus(ctx, task, iteration)
|
||||
return iteration
|
||||
}
|
||||
|
||||
@@ -301,7 +347,8 @@ func (sm *SubagentManager) executeTask(ctx context.Context, task *SubagentTask)
|
||||
argsJSON, _ := json.Marshal(tc.Arguments)
|
||||
toolStart := time.Now().UTC()
|
||||
toolSpanID := sm.emitToolSpanStart(subTraceCtx, toolStart, tc.Name, tc.ID, string(argsJSON))
|
||||
result := toolsReg.Execute(ctx, tc.Name, tc.Arguments)
|
||||
result := toolsReg.Execute(toolCtx, tc.Name, tc.Arguments)
|
||||
result.ForLLM = tracing.RedactText(ctx, result.ForLLM)
|
||||
sm.emitToolSpanEnd(subTraceCtx, toolSpanID, toolStart, result.ForLLM, result.IsError)
|
||||
|
||||
// Capture media file paths from tool results (e.g. image generation).
|
||||
@@ -327,19 +374,19 @@ func (sm *SubagentManager) executeTask(ctx context.Context, task *SubagentTask)
|
||||
}
|
||||
|
||||
sm.mu.Lock()
|
||||
if finalContent == "" {
|
||||
finalContent = "Task completed but no final response was generated."
|
||||
if task.Status != TaskStatusCancelled {
|
||||
if finalContent == "" {
|
||||
finalContent = "Task completed but no final response was generated."
|
||||
}
|
||||
finalContent = tracing.RedactText(ctx, finalContent)
|
||||
task.Status = TaskStatusCompleted
|
||||
task.Result = finalContent
|
||||
task.Media = mediaFiles
|
||||
}
|
||||
task.Status = TaskStatusCompleted
|
||||
task.Result = finalContent
|
||||
task.Media = mediaFiles
|
||||
sm.mu.Unlock()
|
||||
|
||||
slog.Info("subagent completed", "id", task.ID, "iterations", iteration)
|
||||
|
||||
// Persist final status to DB (fire-and-forget).
|
||||
go sm.persistStatus(ctx, task, iteration)
|
||||
|
||||
return iteration
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user