fix(collaboration): harden delegated task isolation (#1486)

* feat(collaboration): isolate delegated artifacts and child runs

Isolate delegated inputs and outputs behind secure artifact exchange lifecycles. Scope Agent Link tasks by tenant and root agent, and enforce delegation spawn-tree boundaries. Add process-wide child-run admission and preserve logical media paths across native, MCP, and sandbox execution.

* fix(collaboration): harden delegated task isolation

Enforce tenant and root-agent task scope across migrations and stores. Add exactly-once async completion delivery, delegated sandbox boundaries, and confined artifact and media recovery across runtime surfaces.

* fix(collaboration): recover interrupted async tasks

* fix(collaboration): normalize persisted child-run status

---------

Co-authored-by: ntduc <ntduc@cpp.ai.vn>

cmd/gateway.go

@@ -43,6 +43,7 @@ import (
 	mcpbridge "github.com/nextlevelbuilder/goclaw/internal/mcp"
 	mcpoauth "github.com/nextlevelbuilder/goclaw/internal/mcp/oauth"
 	"github.com/nextlevelbuilder/goclaw/internal/media"
+	"github.com/nextlevelbuilder/goclaw/internal/orchestration"
 	"github.com/nextlevelbuilder/goclaw/internal/providers"
 	"github.com/nextlevelbuilder/goclaw/internal/scheduler"
 	"github.com/nextlevelbuilder/goclaw/internal/security"
@@ -98,6 +99,29 @@ type snapshotBucketRefresher interface {
 	RefreshBuckets(context.Context, []time.Time) (int, error)
 }
 
+func recoverInterruptedSubagentTasks(
+	ctx context.Context,
+	stores *store.Stores,
+	retryDelay time.Duration,
+) (int64, error) {
+	recoveryCtx := store.WithTenantID(ctx, store.MasterTenantID)
+	for {
+		recovered, err := stores.SubagentTaskRecovery.RecoverInterrupted(recoveryCtx)
+		if err == nil {
+			return recovered, nil
+		}
+		slog.Warn("subagent_tasks.recover_interrupted_retrying", "err", err)
+
+		timer := time.NewTimer(retryDelay)
+		select {
+		case <-ctx.Done():
+			timer.Stop()
+			return 0, ctx.Err()
+		case <-timer.C:
+		}
+	}
+}
+
 func backfillTraceCostsAfterPricingSync(ctx context.Context, stores *store.Stores, snapshots snapshotBucketRefresher) {
 	if stores == nil {
 		return
@@ -292,6 +316,24 @@ func runGateway() {
 		}
 	}
 
+	// Accepted async child runs are process-owned and cannot resume after a
+	// restart. Reconcile their durable rows before wiring tools or accepting
+	// traffic so completion lookups never remain queued/running forever.
+	if pgStores.SubagentTaskRecovery != nil {
+		startupCtx, stopStartup := signal.NotifyContext(
+			context.Background(), syscall.SIGINT, syscall.SIGTERM,
+		)
+		n, err := recoverInterruptedSubagentTasks(startupCtx, pgStores, time.Second)
+		stopStartup()
+		if err != nil {
+			slog.Info("subagent_tasks.recover_interrupted_aborted", "err", err)
+			return
+		}
+		if n > 0 {
+			slog.Info("subagent_tasks.recover_interrupted", "count", n)
+		}
+	}
+
 	if traceCollector != nil {
 		defer traceCollector.Stop()
 		// OTel OTLP export: compiled via build tags. Build with 'go build -tags otel' to enable.
@@ -456,7 +498,8 @@ func runGateway() {
 	}
 
 	// Subagent system (secureCLI store wired so subagent ExecTools enforce the gate)
-	subagentMgr := setupSubagents(providerRegistry, cfg, msgBus, toolsReg, workspace, sandboxMgr, pgStores.SecureCLI, usageCapSvc)
+	childRunAdmission := orchestration.NewChildRunAdmission(edition.Current().ChildRunLimit(), 128)
+	subagentMgr := setupSubagents(providerRegistry, cfg, msgBus, toolsReg, workspace, sandboxMgr, pgStores.SecureCLI, usageCapSvc, childRunAdmission)
 	if subagentMgr != nil {
 		// Wire announce queue for batched subagent result delivery (matching TS debounce pattern).
 		announceQueue := tools.NewAnnounceQueue(1000, 20, makeDelegateAnnounceCallback(subagentMgr, msgBus))
@@ -577,7 +620,7 @@ func runGateway() {
 	var mcpPool *mcpbridge.Pool
 	var mediaStore *media.Store
 	var postTurn tools.PostTurnProcessor
-	contextFileInterceptor, mcpPool, mediaStore, postTurn = wireExtras(pgStores, agentRouter, providerRegistry, modelReg, msgBus, pgStores.Sessions, toolsReg, toolPE, skillsLoader, hasMemory, traceCollector, workspace, cfg.Gateway.InjectionAction, cfg, sandboxMgr, redisClient, domainBus, usageCapSvc, mcpOAuthRefresher)
+	contextFileInterceptor, mcpPool, mediaStore, postTurn = wireExtras(pgStores, agentRouter, providerRegistry, modelReg, msgBus, pgStores.Sessions, toolsReg, toolPE, skillsLoader, hasMemory, traceCollector, workspace, cfg.Gateway.InjectionAction, cfg, sandboxMgr, redisClient, domainBus, usageCapSvc, mcpOAuthRefresher, childRunAdmission)
 	if mcpPool != nil {
 		defer mcpPool.Stop()
 	}
@@ -1042,9 +1085,11 @@ func runGateway() {
 		sandboxMgr:        sandboxMgr,
 		postTurn:          postTurn,
 		subagentMgr:       subagentMgr,
+		childRunAdmission: childRunAdmission,
 		consumerTeamStore: consumerTeamStore,
 		auditCh:           auditCh,
 		sigCh:             sigCh,
+		terminateProcess:  os.Exit,
 	})
 }
 

cmd/gateway_agents.go

@@ -11,6 +11,7 @@ import (
 	"github.com/nextlevelbuilder/goclaw/internal/bus"
 	"github.com/nextlevelbuilder/goclaw/internal/config"
 	"github.com/nextlevelbuilder/goclaw/internal/memory"
+	"github.com/nextlevelbuilder/goclaw/internal/orchestration"
 	"github.com/nextlevelbuilder/goclaw/internal/providers"
 	"github.com/nextlevelbuilder/goclaw/internal/sandbox"
 	"github.com/nextlevelbuilder/goclaw/internal/store"
@@ -163,7 +164,7 @@ func buildEmbeddingProvider(
 	return nil
 }
 
-func setupSubagents(providerReg *providers.Registry, cfg *config.Config, msgBus *bus.MessageBus, toolsReg *tools.Registry, workspace string, sandboxMgr sandbox.Manager, secureCLIStore store.SecureCLIStore, usageCapSvc *usagecaps.Service) *tools.SubagentManager {
+func setupSubagents(providerReg *providers.Registry, cfg *config.Config, msgBus *bus.MessageBus, toolsReg *tools.Registry, workspace string, sandboxMgr sandbox.Manager, secureCLIStore store.SecureCLIStore, usageCapSvc *usagecaps.Service, admission *orchestration.ChildRunAdmission) *tools.SubagentManager {
 	names := providerReg.List(context.Background())
 	if len(names) == 0 {
 		return nil
@@ -211,7 +212,7 @@ func setupSubagents(providerReg *providers.Registry, cfg *config.Config, msgBus
 		return reg
 	}
 
-	manager := tools.NewSubagentManager(provider, providerReg, agentCfg.Model, msgBus, toolsFactory, subCfg)
+	manager := tools.NewSubagentManagerWithAdmission(provider, providerReg, agentCfg.Model, msgBus, toolsFactory, subCfg, admission)
 	manager.SetUsageCapService(usageCapSvc)
 	manager.SetAgentBudget(agentCfg.ContextWindow, agentCfg.MaxTokens)
 	return manager

cmd/gateway_child_run_lifecycle_test.go

@@ -0,0 +1,52 @@
+package cmd
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+
+	"github.com/nextlevelbuilder/goclaw/internal/orchestration"
+	"github.com/nextlevelbuilder/goclaw/internal/store"
+)
+
+func TestDrainChildRunsWithRetryReturnsTypedFailure(t *testing.T) {
+	admission := orchestration.NewChildRunAdmission(1, 2)
+	started := make(chan struct{})
+	release := make(chan struct{})
+	ticket, err := admission.Enqueue(context.Background(), orchestration.ChildRunConstraints{
+		TenantID: store.MasterTenantID,
+		TaskID:   uuid.NewString(),
+	}, func(context.Context, *orchestration.ChildRunLease) {
+		close(started)
+		<-release
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := ticket.Activate(); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case <-started:
+	case <-time.After(time.Second):
+		t.Fatal("child run did not start")
+	}
+
+	err = drainChildRunsWithRetry(admission, 10*time.Millisecond, 10*time.Millisecond)
+	if !errors.Is(err, orchestration.ErrChildRunDrainTimeout) {
+		t.Fatalf("drain error = %v, want typed timeout", err)
+	}
+
+	close(release)
+	select {
+	case <-ticket.Done():
+	case <-time.After(time.Second):
+		t.Fatal("child run did not finish after forced termination signal")
+	}
+	if err := admission.Close(context.Background()); err != nil {
+		t.Fatalf("final drain: %v", err)
+	}
+}

cmd/gateway_consumer_handlers.go

@@ -79,6 +79,7 @@ func handleSubagentAnnounce(
 	if sid := msg.Metadata[tools.MetaOriginRootSpanID]; sid != "" {
 		parentRootSpanID, _ = uuid.Parse(sid)
 	}
+	rootAgentID, _ := uuid.Parse(msg.Metadata[tools.MetaSubagentRootAgentID])
 
 	// Group-scoped UserID for subagent announce (same logic as main lane).
 	announceUserID := msg.UserID
@@ -116,9 +117,7 @@ func handleSubagentAnnounce(
 	originSenderID := msg.Metadata[tools.MetaOriginSenderID]
 	originRole := msg.Metadata[tools.MetaOriginRole]
 
-	queueKey := fmt.Sprintf("%s:%s", msg.TenantID, sessionKey)
 	routing := subagentAnnounceRouting{
-		QueueKey:         queueKey,
 		SessionKey:       sessionKey,
 		TenantID:         msg.TenantID,
 		OrigChannel:      origChannel,
@@ -130,10 +129,16 @@ func handleSubagentAnnounce(
 		SenderID:         originSenderID,
 		Role:             originRole,
 		ParentAgent:      parentAgent,
+		RootAgentID:      rootAgentID,
 		ParentTraceID:    parentTraceID,
 		ParentRootSpanID: parentRootSpanID,
 		OutMeta:          buildAnnounceOutMeta(origLocalKey),
 	}
+	// Batch only announces with identical routing and authority. A session can
+	// be shared by multiple group senders or local topic keys; using only
+	// tenant+session would let the first item's sender/role govern the rest.
+	queueKey := subagentAnnounceRoutingKey(routing)
+	routing.QueueKey = queueKey
 
 	// Enqueue into producer-consumer queue using tenant-scoped key from routing.
 	isProcessor := enqueueSubagentAnnounce(queueKey, entry)
@@ -142,7 +147,9 @@ func handleSubagentAnnounce(
 			defer safego.Recover(nil, "component", "subagent_announce_loop", "session", sessionKey)
 
 			// Fetch live roster for merged announce context.
-			roster := deps.SubagentMgr.RosterForParent(parentAgent)
+			roster := deps.SubagentMgr.RosterForParent(tools.TaskScope{
+				TenantID: msg.TenantID, RootAgentID: rootAgentID, RootAgentKey: parentAgent,
+			})
 
 			processSubagentAnnounceLoop(ctx, routing, roster, deps.SubagentMgr, deps.Sched, deps.MsgBus, deps.Cfg, deps.ChannelMgr)
 		})
@@ -151,6 +158,22 @@ func handleSubagentAnnounce(
 	return true
 }
 
+func subagentAnnounceRoutingKey(r subagentAnnounceRouting) string {
+	return strings.Join([]string{
+		r.TenantID.String(),
+		r.RootAgentID.String(),
+		r.ParentAgent,
+		r.SessionKey,
+		r.OrigChannel,
+		r.OrigChatID,
+		r.OrigPeerKind,
+		r.OrigLocalKey,
+		r.UserID,
+		r.SenderID,
+		r.Role,
+	}, "\x00")
+}
+
 // handleTeammateMessage processes teammate messages: bypass debounce, route to target
 // agent session using the "team" lane, then announce result back to lead.
 // Returns true if the message was handled (caller should continue).
@@ -560,11 +583,14 @@ func buildTeammateAnnounce(ctx context.Context, outcome scheduler.RunOutcome, se
 	} else if outcome.Result == nil {
 		slog.Warn("teammate message: nil result without error", "from", senderID)
 		return "", nil, false
-	} else if (outcome.Result.Content == "" && len(outcome.Result.Media) == 0) || agent.IsSilentReply(outcome.Result.Content) {
+	} else if normalized, shouldDeliver := normalizeAgentOutboundContent(
+		outcome.Result.Content,
+		len(outcome.Result.Media),
+	); !shouldDeliver {
 		slog.Info("teammate message: suppressed silent/empty reply", "from", senderID)
 		return "", nil, false
 	} else {
-		content = outcome.Result.Content
+		content = normalized
 		media = outcome.Result.Media
 	}
 

cmd/gateway_consumer_normal.go

@@ -575,9 +575,14 @@ func processNormalMessage(
 			return
 		}
 
-		// Suppress empty/NO_REPLY responses (matching TS normalize-reply.ts).
-		// Still publish an empty outbound so channels can clean up placeholder/thinking indicators.
-		if outcome.Result.Content == "" || agent.IsSilentReply(outcome.Result.Content) {
+		// Suppress silent text only when the result has no media. A tool or
+		// delegate may legitimately return NO_REPLY with an attached artifact;
+		// that must continue as a media-only outbound message.
+		resultContent, shouldDeliver := normalizeAgentOutboundContent(
+			outcome.Result.Content,
+			len(outcome.Result.Media),
+		)
+		if !shouldDeliver {
 			slog.Info("inbound: suppressed silent/empty reply",
 				"channel", channel,
 				"chat_id", chatID,
@@ -615,7 +620,7 @@ func processNormalMessage(
 		// Sanitize voice agent replies: replace technical errors with user-friendly fallback.
 		replyContent := voiceguard.SanitizeReply(
 			deps.Cfg.Channels.Telegram.VoiceAgentID, agentKey,
-			channel, peerKind, inboundContent, outcome.Result.Content,
+			channel, peerKind, inboundContent, resultContent,
 			deps.Cfg.Channels.Telegram.AudioGuardFallbackTranscript,
 			deps.Cfg.Channels.Telegram.AudioGuardFallbackNoTranscript,
 			deps.Cfg.Channels.Telegram.AudioGuardErrorMarkers,

cmd/gateway_consumer_outbound.go

@@ -0,0 +1,16 @@
+package cmd
+
+import "github.com/nextlevelbuilder/goclaw/internal/agent"
+
+// normalizeAgentOutboundContent keeps silent text out of user-facing messages
+// while allowing attached media to continue through channel delivery.
+func normalizeAgentOutboundContent(content string, mediaCount int) (string, bool) {
+	isSilent := content == "" || agent.IsSilentReply(content)
+	if !isSilent {
+		return content, true
+	}
+	if mediaCount == 0 {
+		return "", false
+	}
+	return "", true
+}

cmd/gateway_delegate.go

@@ -0,0 +1,54 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/google/uuid"
+
+	"github.com/nextlevelbuilder/goclaw/internal/agent"
+	"github.com/nextlevelbuilder/goclaw/internal/bus"
+	"github.com/nextlevelbuilder/goclaw/internal/sandbox"
+	"github.com/nextlevelbuilder/goclaw/internal/tools"
+)
+
+// buildAgentLinkRunRequest preserves the origin's authorization-bearing
+// identity while keeping delegation on its internal delivery channel.
+func buildAgentLinkRunRequest(req tools.DelegateRequest, sessionKey string) agent.RunRequest {
+	return agent.RunRequest{
+		RunID:               uuid.New().String(),
+		SessionKey:          sessionKey,
+		Message:             req.Task,
+		UserID:              req.UserID,
+		SenderID:            req.SenderID,
+		Role:                req.Role,
+		Channel:             "delegate",
+		ChannelType:         req.ChannelType,
+		ChatID:              req.ChatID,
+		PeerKind:            req.PeerKind,
+		RunKind:             "delegate",
+		DelegationID:        req.DelegationID,
+		ParentAgentID:       req.FromAgentKey,
+		WorkspaceChannel:    req.Channel,
+		WorkspaceChatID:     req.ChatID,
+		DelegateInputsPath:  req.DelegateInputsPath,
+		DelegateOutputsPath: req.DelegateOutputsPath,
+	}
+}
+
+// Agent Link artifacts are published from the exchange manifest. Raw delegate
+// media paths point into B's ephemeral workspace and must never cross back to A.
+func agentMediaToBusFiles(_ []agent.MediaResult) []bus.MediaFile { return nil }
+
+func releaseDelegationSandbox(ctx context.Context, manager sandbox.Manager, sessionKey string) error {
+	if manager == nil {
+		return nil
+	}
+	cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
+	defer cancel()
+	if err := manager.Release(cleanupCtx, sessionKey); err != nil {
+		return fmt.Errorf("delegation sandbox release failed")
+	}
+	return nil
+}

cmd/gateway_delegate_test.go

@@ -0,0 +1,109 @@
+package cmd
+
+import (
+	"context"
+	"errors"
+	"path/filepath"
+	"testing"
+
+	"github.com/google/uuid"
+
+	"github.com/nextlevelbuilder/goclaw/internal/agent"
+	"github.com/nextlevelbuilder/goclaw/internal/sandbox"
+	"github.com/nextlevelbuilder/goclaw/internal/tools"
+)
+
+type delegationReleaseManager struct {
+	released string
+	err      error
+}
+
+func (m *delegationReleaseManager) Get(context.Context, string, string, *sandbox.Config, ...sandbox.GetOption) (sandbox.Sandbox, error) {
+	return nil, errors.New("not implemented")
+}
+func (m *delegationReleaseManager) Release(_ context.Context, key string) error {
+	m.released = key
+	return m.err
+}
+func (*delegationReleaseManager) ReleaseAll(context.Context) error { return nil }
+func (*delegationReleaseManager) Stop()                            {}
+func (*delegationReleaseManager) Stats() map[string]any            { return nil }
+
+func TestBuildAgentLinkRunRequestPreservesGroupAuthorizationScope(t *testing.T) {
+	req := tools.DelegateRequest{
+		FromAgentKey:        "coordinator",
+		Task:                "create the report",
+		DelegationID:        uuid.NewString(),
+		UserID:              "group:telegram:-100123",
+		SenderID:            "386246614",
+		Role:                "viewer",
+		Channel:             "telegram-main",
+		ChannelType:         "telegram",
+		ChatID:              "-100123",
+		PeerKind:            "group",
+		DelegateInputsPath:  filepath.Join(t.TempDir(), "inputs"),
+		DelegateOutputsPath: filepath.Join(t.TempDir(), "outputs"),
+	}
+
+	got := buildAgentLinkRunRequest(req, "delegate:session")
+
+	if got.UserID != req.UserID || got.SenderID != req.SenderID || got.Role != req.Role {
+		t.Fatalf("authorization scope = (%q, %q, %q), want (%q, %q, %q)",
+			got.UserID, got.SenderID, got.Role, req.UserID, req.SenderID, req.Role)
+	}
+	if got.Channel != "delegate" || got.ChannelType != req.ChannelType {
+		t.Fatalf("channel = (%q, %q), want (delegate, %q)", got.Channel, got.ChannelType, req.ChannelType)
+	}
+	if got.ChatID != req.ChatID || got.PeerKind != req.PeerKind {
+		t.Fatalf("chat scope = (%q, %q), want (%q, %q)",
+			got.ChatID, got.PeerKind, req.ChatID, req.PeerKind)
+	}
+	if got.WorkspaceChannel != req.Channel || got.WorkspaceChatID != req.ChatID {
+		t.Fatalf("workspace scope = (%q, %q), want (%q, %q)",
+			got.WorkspaceChannel, got.WorkspaceChatID, req.Channel, req.ChatID)
+	}
+	if got.RunID == "" || got.RunKind != "delegate" || got.DelegationID != req.DelegationID {
+		t.Fatalf("delegate classification = %#v", got)
+	}
+	if got.DelegateInputsPath != req.DelegateInputsPath ||
+		got.DelegateOutputsPath != req.DelegateOutputsPath {
+		t.Fatalf("artifact runtime wiring = (%q, %q), want (%q, %q)",
+			got.DelegateInputsPath, got.DelegateOutputsPath,
+			req.DelegateInputsPath, req.DelegateOutputsPath)
+	}
+	if got.Media != nil {
+		t.Fatalf("run media = %#v, want exchange-only input delivery", got.Media)
+	}
+}
+
+func TestAgentMediaToBusFilesDiscardsEphemeralDelegateMedia(t *testing.T) {
+	got := agentMediaToBusFiles([]agent.MediaResult{{
+		Path:        filepath.Join(t.TempDir(), "ephemeral.png"),
+		ContentType: "image/png",
+	}})
+	if got != nil {
+		t.Fatalf("media = %#v, want raw delegate media discarded", got)
+	}
+}
+
+func TestReleaseDelegationSandboxUsesExactSessionKey(t *testing.T) {
+	manager := &delegationReleaseManager{}
+	const sessionKey = "delegate:from:target:4fe8220f-07f1-4e64-a95c-b49ebc39db4a"
+	if err := releaseDelegationSandbox(context.Background(), manager, sessionKey); err != nil {
+		t.Fatalf("releaseDelegationSandbox: %v", err)
+	}
+	if manager.released != sessionKey {
+		t.Fatalf("released key = %q, want %q", manager.released, sessionKey)
+	}
+}
+
+func TestReleaseDelegationSandboxRedactsManagerFailure(t *testing.T) {
+	manager := &delegationReleaseManager{err: errors.New("/private/exchange/inputs remained mounted")}
+	err := releaseDelegationSandbox(context.Background(), manager, "delegate:session")
+	if err == nil {
+		t.Fatal("releaseDelegationSandbox unexpectedly succeeded")
+	}
+	if got := err.Error(); got != "delegation sandbox release failed" {
+		t.Fatalf("error = %q, want redacted release failure", got)
+	}
+}

cmd/gateway_lifecycle.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"context"
+	"fmt"
 	"log/slog"
 	"os"
 	"strings"
@@ -14,6 +15,7 @@ import (
 	"github.com/nextlevelbuilder/goclaw/internal/config"
 	"github.com/nextlevelbuilder/goclaw/internal/edition"
 	"github.com/nextlevelbuilder/goclaw/internal/heartbeat"
+	"github.com/nextlevelbuilder/goclaw/internal/orchestration"
 	"github.com/nextlevelbuilder/goclaw/internal/sandbox"
 	"github.com/nextlevelbuilder/goclaw/internal/scheduler"
 	"github.com/nextlevelbuilder/goclaw/internal/store"
@@ -33,9 +35,71 @@ type lifecycleDeps struct {
 	sandboxMgr        sandbox.Manager
 	postTurn          tools.PostTurnProcessor
 	subagentMgr       *tools.SubagentManager
+	childRunAdmission *orchestration.ChildRunAdmission
 	consumerTeamStore store.TeamStore
 	auditCh           chan bus.AuditEventPayload
 	sigCh             chan os.Signal
+	terminateProcess  func(int)
+}
+
+func drainChildRunsWithRetry(
+	admission *orchestration.ChildRunAdmission,
+	firstTimeout time.Duration,
+	retryTimeout time.Duration,
+) error {
+	if admission == nil {
+		return nil
+	}
+	for attempt, timeout := range []time.Duration{firstTimeout, retryTimeout} {
+		drainCtx, drainCancel := context.WithTimeout(context.Background(), timeout)
+		err := admission.Close(drainCtx)
+		drainCancel()
+		if err == nil {
+			return nil
+		}
+		slog.Error("gateway: child-run drain attempt failed",
+			"attempt", attempt+1, "timeout", timeout, "error", err)
+	}
+	return fmt.Errorf("%w after retry", orchestration.ErrChildRunDrainTimeout)
+}
+
+func drainSubagentManagerWithRetry(
+	manager *tools.SubagentManager,
+	firstTimeout time.Duration,
+	retryTimeout time.Duration,
+) error {
+	if manager == nil {
+		return nil
+	}
+	for attempt, timeout := range []time.Duration{firstTimeout, retryTimeout} {
+		drainCtx, drainCancel := context.WithTimeout(context.Background(), timeout)
+		err := manager.CloseContext(drainCtx)
+		drainCancel()
+		if err == nil {
+			return nil
+		}
+		slog.Error("gateway: subagent lifecycle drain attempt failed",
+			"attempt", attempt+1, "timeout", timeout, "error", err)
+	}
+	return fmt.Errorf("%w after retry", tools.ErrSubagentLifecycleDrainTimeout)
+}
+
+func drainDelegateToolWithRetry(
+	tool interface{ CloseContext(context.Context) error },
+	firstTimeout time.Duration,
+	retryTimeout time.Duration,
+) error {
+	for attempt, timeout := range []time.Duration{firstTimeout, retryTimeout} {
+		drainCtx, drainCancel := context.WithTimeout(context.Background(), timeout)
+		err := tool.CloseContext(drainCtx)
+		drainCancel()
+		if err == nil {
+			return nil
+		}
+		slog.Error("gateway: delegate completion drain attempt failed",
+			"attempt", attempt+1, "timeout", timeout, "error", err)
+	}
+	return fmt.Errorf("delegate completion drain failed after retry")
 }
 
 // runLifecycle wires config-reload subscribers, starts consumers, task recovery,
@@ -198,6 +262,20 @@ func (d *gatewayDeps) runLifecycle(
 		// Broadcast shutdown event
 		d.server.BroadcastEvent(*protocol.NewEvent(protocol.EventShutdown, nil))
 
+		// Close child-run intake first. A drain timeout must terminate without
+		// unwinding runGateway defers under a still-live child callback.
+		if deps.childRunAdmission != nil {
+			if err := drainChildRunsWithRetry(deps.childRunAdmission, 30*time.Second, 5*time.Second); err != nil {
+				slog.Error("gateway: terminating after child-run drain failure", "error", err)
+				terminate := deps.terminateProcess
+				if terminate == nil {
+					terminate = os.Exit
+				}
+				terminate(1)
+				return
+			}
+		}
+
 		// Stop channels, cron, heartbeat, and task ticker
 		d.channelMgr.StopAll(context.Background())
 		d.pgStores.Cron.Stop()
@@ -216,6 +294,35 @@ func (d *gatewayDeps) runLifecycle(
 			close(deps.auditCh)
 		}
 
+		if delegate, ok := d.toolsReg.Get("delegate"); ok {
+			if closer, ok := delegate.(interface {
+				CloseContext(context.Context) error
+			}); ok {
+				if err := drainDelegateToolWithRetry(closer, 65*time.Second, 10*time.Second); err != nil {
+					slog.Error("gateway: terminating after delegate completion drain failure", "error", err)
+					terminate := deps.terminateProcess
+					if terminate == nil {
+						terminate = os.Exit
+					}
+					terminate(1)
+					return
+				}
+			} else if closer, ok := delegate.(interface{ Close() }); ok {
+				closer.Close()
+			}
+		}
+		if deps.subagentMgr != nil {
+			if err := drainSubagentManagerWithRetry(deps.subagentMgr, 65*time.Second, 10*time.Second); err != nil {
+				slog.Error("gateway: terminating after subagent lifecycle drain failure", "error", err)
+				terminate := deps.terminateProcess
+				if terminate == nil {
+					terminate = os.Exit
+				}
+				terminate(1)
+				return
+			}
+		}
+
 		// Close provider resources (e.g. Claude CLI temp files)
 		d.providerRegistry.Close()
 

cmd/gateway_managed.go

@@ -62,6 +62,7 @@ func wireExtras(
 	domainBus eventbus.DomainEventBus,
 	usageCapSvc *usagecaps.Service,
 	mcpOAuthProvider mcpbridge.OAuthTokenProvider, // nil = OAuth injection disabled
+	childRunAdmission *orchestration.ChildRunAdmission,
 ) (*tools.ContextFileInterceptor, *mcpbridge.Pool, *media.Store, tools.PostTurnProcessor) {
 	// 1. Build cache instances (in-memory or Redis depending on build tags)
 	agentCtxCache, userCtxCache := makeCaches(redisClient)
@@ -450,25 +451,32 @@ func wireExtras(
 			// Link delegate trace to parent trace
 			delegateCtx := tracing.WithDelegateParentTraceID(ctx, tracing.TraceIDFromContext(ctx))
 
-			runReq := agent.RunRequest{
-				RunID:         uuid.New().String(),
-				SessionKey:    sessionKey,
-				Message:       req.Task,
-				UserID:        req.UserID,
-				Channel:       "delegate",
-				RunKind:       "delegate",
-				DelegationID:  req.DelegationID,
-				ParentAgentID: req.FromAgentKey,
+			runReq := buildAgentLinkRunRequest(req, sessionKey)
+			var delegateTraceID uuid.UUID
+			runReq.OnTraceCreated = func(traceID uuid.UUID) {
+				delegateTraceID = traceID
+				if req.OnTraceCreated != nil {
+					req.OnTraceCreated(traceID)
+				}
 			}
-			result, err := loop.Run(delegateCtx, runReq)
-			if err != nil {
-				return tools.DelegateResult{}, err
+			result, runErr := loop.Run(delegateCtx, runReq)
+			if releaseErr := releaseDelegationSandbox(ctx, sandboxMgr, sessionKey); releaseErr != nil {
+				return tools.DelegateResult{TraceID: delegateTraceID}, releaseErr
 			}
-			cr := orchestration.CaptureFromRunResult(result, 0)
-			return tools.DelegateResult{Content: cr.Content, Media: cr.Media}, nil
+			if runErr != nil {
+				return tools.DelegateResult{TraceID: delegateTraceID}, runErr
+			}
+			return tools.DelegateResult{
+				Content: result.Content,
+				Media:   agentMediaToBusFiles(result.Media),
+				TraceID: delegateTraceID,
+			}, nil
 		}
-		delegateTool := tools.NewDelegateTool(stores.AgentLinks, stores.Agents, domainBus, delegateRunFn)
+		delegateTool := tools.NewDelegateToolWithAdmission(stores.AgentLinks, stores.Agents, domainBus, delegateRunFn, childRunAdmission)
+		delegateTool.SetDataDir(appCfg.DataDir)
+		delegateTool.SetWorkspace(workspace)
 		delegateTool.SetMsgBus(msgBus)
+		delegateTool.SetTaskStore(stores.SubagentTasks)
 		delegateTool.SetHookDispatcher(hookDispatcher)
 		toolsReg.Register(delegateTool)
 		slog.Info("delegate tool wired")

cmd/gateway_subagent_announce_key_test.go

@@ -0,0 +1,47 @@
+package cmd
+
+import (
+	"testing"
+
+	"github.com/google/uuid"
+)
+
+func TestSubagentAnnounceRoutingKeyScopesRoutingAndAuthority(t *testing.T) {
+	base := subagentAnnounceRouting{
+		TenantID:     uuid.New(),
+		RootAgentID:  uuid.New(),
+		ParentAgent:  "root",
+		SessionKey:   "session",
+		OrigChannel:  "telegram",
+		OrigChatID:   "chat",
+		OrigPeerKind: "group",
+		OrigLocalKey: "topic",
+		UserID:       "user",
+		SenderID:     "sender",
+		Role:         "operator",
+	}
+	want := subagentAnnounceRoutingKey(base)
+
+	cases := map[string]func(*subagentAnnounceRouting){
+		"tenant":       func(v *subagentAnnounceRouting) { v.TenantID = uuid.New() },
+		"root id":      func(v *subagentAnnounceRouting) { v.RootAgentID = uuid.New() },
+		"parent agent": func(v *subagentAnnounceRouting) { v.ParentAgent += "-other" },
+		"session":      func(v *subagentAnnounceRouting) { v.SessionKey += "-other" },
+		"channel":      func(v *subagentAnnounceRouting) { v.OrigChannel += "-other" },
+		"chat":         func(v *subagentAnnounceRouting) { v.OrigChatID += "-other" },
+		"peer kind":    func(v *subagentAnnounceRouting) { v.OrigPeerKind += "-other" },
+		"local key":    func(v *subagentAnnounceRouting) { v.OrigLocalKey += "-other" },
+		"user":         func(v *subagentAnnounceRouting) { v.UserID += "-other" },
+		"sender":       func(v *subagentAnnounceRouting) { v.SenderID += "-other" },
+		"role":         func(v *subagentAnnounceRouting) { v.Role += "-other" },
+	}
+	for name, mutate := range cases {
+		t.Run(name, func(t *testing.T) {
+			got := base
+			mutate(&got)
+			if key := subagentAnnounceRoutingKey(got); key == want {
+				t.Fatalf("routing key did not change when %s changed", name)
+			}
+		})
+	}
+}

cmd/gateway_subagent_announce_queue.go

@@ -27,7 +27,9 @@ func makeDelegateAnnounceCallback(
 	msgBus *bus.MessageBus,
 ) func(sessionKey string, items []tools.AnnounceQueueItem, meta tools.AnnounceMetadata) {
 	return func(sessionKey string, items []tools.AnnounceQueueItem, meta tools.AnnounceMetadata) {
-		roster := subagentMgr.RosterForParent(meta.ParentAgent)
+		roster := subagentMgr.RosterForParent(tools.TaskScope{
+			TenantID: meta.OriginTenantID, RootAgentID: meta.RootAgentID, RootAgentKey: meta.ParentAgent,
+		})
 		content := tools.FormatBatchedAnnounce(items, roster)
 		senderID := fmt.Sprintf("subagent:batch-%d", len(items))
 		label := items[0].Label
@@ -35,12 +37,13 @@ func makeDelegateAnnounceCallback(
 			label = fmt.Sprintf("%d tasks", len(items))
 		}
 		batchMeta := map[string]string{
-			tools.MetaOriginChannel:    meta.OriginChannel,
-			tools.MetaOriginPeerKind:   meta.OriginPeerKind,
-			tools.MetaParentAgent:      meta.ParentAgent,
-			tools.MetaSubagentLabel:    label,
-			tools.MetaOriginTraceID:    meta.OriginTraceID,
-			tools.MetaOriginRootSpanID: meta.OriginRootSpanID,
+			tools.MetaOriginChannel:       meta.OriginChannel,
+			tools.MetaOriginPeerKind:      meta.OriginPeerKind,
+			tools.MetaParentAgent:         meta.ParentAgent,
+			tools.MetaSubagentRootAgentID: meta.RootAgentID.String(),
+			tools.MetaSubagentLabel:       label,
+			tools.MetaOriginTraceID:       meta.OriginTraceID,
+			tools.MetaOriginRootSpanID:    meta.OriginRootSpanID,
 		}
 		if meta.OriginLocalKey != "" {
 			batchMeta[tools.MetaOriginLocalKey] = meta.OriginLocalKey
@@ -69,7 +72,7 @@ func makeDelegateAnnounceCallback(
 			"tasks":   len(items),
 		})
 
-		msgBus.PublishInbound(bus.InboundMessage{
+		delivered := tools.PublishAsyncCompletion(context.Background(), msgBus, bus.InboundMessage{
 			Channel:  "system",
 			SenderID: senderID,
 			ChatID:   meta.OriginChatID,
@@ -79,6 +82,30 @@ func makeDelegateAnnounceCallback(
 			Metadata: batchMeta,
 			Media:    batchMedia,
 		})
+		for _, item := range items {
+			if !item.DurablyPersisted {
+				slog.Error("subagent.batch_announce_without_durable_terminal",
+					"task_id", item.SubagentID,
+					"completion_id", item.CompletionID,
+					"root_agent_id", meta.RootAgentID,
+					"delivered", delivered,
+				)
+				continue
+			}
+			subagentMgr.UpdateAnnouncementStatus(
+				store.WithTenantID(context.Background(), meta.OriginTenantID),
+				meta.RootAgentID,
+				item.CompletionID,
+				delivered,
+			)
+		}
+		if !delivered {
+			slog.Warn("subagent.batch_announce_deferred_to_ledger",
+				"root_agent_id", meta.RootAgentID,
+				"batch_size", len(items),
+				"reason", "inbound_bus_full",
+			)
+		}
 	}
 }
 
@@ -96,7 +123,7 @@ type subagentAnnounceEntry struct {
 
 // subagentAnnounceRouting holds shared routing info captured by the first enqueue.
 type subagentAnnounceRouting struct {
-	QueueKey         string    // tenant-scoped key for sync.Map (tenantID:sessionKey)
+	QueueKey         string    // tenant/root/session/topic/user/authority-scoped key
 	SessionKey       string    // original session key (no tenant prefix) for RunRequest
 	TenantID         uuid.UUID // preserved for tenant-scoped scheduling
 	OrigChannel      string
@@ -108,6 +135,7 @@ type subagentAnnounceRouting struct {
 	SenderID         string // real acting sender (preserves permission attribution through re-ingress, #915)
 	Role             string // caller's RBAC role; bypasses per-user grants for admin/operator/owner (#915)
 	ParentAgent      string
+	RootAgentID      uuid.UUID
 	ParentTraceID    uuid.UUID
 	ParentRootSpanID uuid.UUID
 	OutMeta          map[string]string
@@ -156,7 +184,9 @@ func processSubagentAnnounceLoop(
 		}
 
 		// Refresh roster each iteration for up-to-date task statuses.
-		roster = subagentMgr.RosterForParent(r.ParentAgent)
+		roster = subagentMgr.RosterForParent(tools.TaskScope{
+			TenantID: r.TenantID, RootAgentID: r.RootAgentID, RootAgentKey: r.ParentAgent,
+		})
 		content := buildMergedSubagentAnnounce(entries, roster)
 
 		// Collect media from all entries.

cmd/gateway_subagent_announce_queue_test.go

@@ -0,0 +1,89 @@
+package cmd
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+
+	"github.com/nextlevelbuilder/goclaw/internal/bus"
+	"github.com/nextlevelbuilder/goclaw/internal/store"
+	"github.com/nextlevelbuilder/goclaw/internal/tools"
+)
+
+type recordingGatewayTaskStore struct {
+	metadata chan map[string]any
+}
+
+func (*recordingGatewayTaskStore) Create(context.Context, *store.SubagentTaskData) error {
+	return nil
+}
+func (*recordingGatewayTaskStore) Get(context.Context, uuid.UUID, uuid.UUID) (*store.SubagentTaskData, error) {
+	return nil, nil
+}
+func (*recordingGatewayTaskStore) UpdateStatus(context.Context, uuid.UUID, uuid.UUID, string, *string, int, int64, int64) error {
+	return nil
+}
+func (*recordingGatewayTaskStore) ListByParent(context.Context, uuid.UUID, string) ([]store.SubagentTaskData, error) {
+	return nil, nil
+}
+func (*recordingGatewayTaskStore) ListBySession(context.Context, uuid.UUID, string) ([]store.SubagentTaskData, error) {
+	return nil, nil
+}
+func (*recordingGatewayTaskStore) Archive(context.Context, uuid.UUID, time.Duration, int) (int64, error) {
+	return 0, nil
+}
+func (s *recordingGatewayTaskStore) UpdateMetadata(_ context.Context, _ uuid.UUID, _ uuid.UUID, metadata map[string]any) error {
+	s.metadata <- metadata
+	return nil
+}
+
+func TestSubagentBatchAnnouncementDoesNotBlockOnFullInboundBus(t *testing.T) {
+	messageBus := bus.New()
+	for range 1000 {
+		messageBus.PublishInbound(bus.InboundMessage{Content: "fill"})
+	}
+	manager := tools.NewSubagentManager(nil, nil, "", messageBus, nil, tools.SubagentConfig{})
+	taskStore := &recordingGatewayTaskStore{metadata: make(chan map[string]any, 1)}
+	manager.SetTaskStore(taskStore)
+	callback := makeDelegateAnnounceCallback(manager, messageBus)
+	completionID := uuid.New()
+
+	done := make(chan struct{})
+	go func() {
+		callback(
+			"session-1",
+			[]tools.AnnounceQueueItem{{
+				SubagentID:       "task-1",
+				CompletionID:     completionID,
+				DurablyPersisted: true,
+				Label:            "probe",
+				Status:           tools.TaskStatusCompleted,
+				Result:           "done",
+			}},
+			tools.AnnounceMetadata{
+				OriginChatID:     "chat-1",
+				OriginSessionKey: "session-1",
+				OriginTenantID:   uuid.New(),
+				RootAgentID:      uuid.New(),
+				ParentAgent:      "root",
+			},
+		)
+		close(done)
+	}()
+
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatal("batched subagent announcement blocked on a full inbound bus")
+	}
+	select {
+	case metadata := <-taskStore.metadata:
+		if metadata["announcement_status"] != "undelivered" {
+			t.Fatalf("announcement metadata = %#v, want undelivered", metadata)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("batched missed announcement was not recorded after bus saturation")
+	}
+}

cmd/gateway_subagent_recovery_test.go

@@ -0,0 +1,65 @@
+package cmd
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/nextlevelbuilder/goclaw/internal/store"
+)
+
+type scriptedSubagentTaskRecovery struct {
+	mu        sync.Mutex
+	failures  int
+	recovered int64
+	calls     int
+}
+
+func (s *scriptedSubagentTaskRecovery) RecoverInterrupted(ctx context.Context) (int64, error) {
+	if !store.IsMasterScope(ctx) || store.TenantIDFromContext(ctx) != store.MasterTenantID {
+		return 0, errors.New("recovery did not receive explicit master scope")
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.calls++
+	if s.calls <= s.failures {
+		return 0, errors.New("database temporarily unavailable")
+	}
+	return s.recovered, nil
+}
+
+func (s *scriptedSubagentTaskRecovery) callCount() int {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.calls
+}
+
+func TestRecoverInterruptedSubagentTasksRetriesBeforeTraffic(t *testing.T) {
+	recovery := &scriptedSubagentTaskRecovery{failures: 2, recovered: 3}
+	stores := &store.Stores{SubagentTaskRecovery: recovery}
+	recovered, err := recoverInterruptedSubagentTasks(
+		context.Background(), stores, time.Millisecond,
+	)
+	if err != nil {
+		t.Fatalf("recoverInterruptedSubagentTasks: %v", err)
+	}
+	if recovered != 3 || recovery.callCount() != 3 {
+		t.Fatalf("recovery = (%d rows, %d calls), want (3, 3)", recovered, recovery.callCount())
+	}
+}
+
+func TestRecoverInterruptedSubagentTasksStopsOnShutdown(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	recovery := &scriptedSubagentTaskRecovery{failures: 1}
+	stores := &store.Stores{SubagentTaskRecovery: recovery}
+	_, err := recoverInterruptedSubagentTasks(ctx, stores, time.Hour)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("recovery error = %v, want context.Canceled", err)
+	}
+	if recovery.callCount() != 1 {
+		t.Fatalf("recovery calls = %d, want 1", recovery.callCount())
+	}
+}