From bb7712a9ffbe41d97d3ed63bb29f19e73a5c61da Mon Sep 17 00:00:00 2001 From: Duc Nguyen <87276974+itsddvn@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:17:40 +0700 Subject: [PATCH] fix(collaboration): harden delegated task isolation (#1486) * feat(collaboration): isolate delegated artifacts and child runs Isolate delegated inputs and outputs behind secure artifact exchange lifecycles. Scope Agent Link tasks by tenant and root agent, and enforce delegation spawn-tree boundaries. Add process-wide child-run admission and preserve logical media paths across native, MCP, and sandbox execution. * fix(collaboration): harden delegated task isolation Enforce tenant and root-agent task scope across migrations and stores. Add exactly-once async completion delivery, delegated sandbox boundaries, and confined artifact and media recovery across runtime surfaces. * fix(collaboration): recover interrupted async tasks * fix(collaboration): normalize persisted child-run status --------- Co-authored-by: ntduc --- .gitignore | 1 + README.md | 6 +- cmd/gateway.go | 49 +- cmd/gateway_agents.go | 5 +- cmd/gateway_child_run_lifecycle_test.go | 52 + cmd/gateway_consumer_handlers.go | 36 +- cmd/gateway_consumer_normal.go | 13 +- cmd/gateway_consumer_outbound.go | 16 + cmd/gateway_delegate.go | 54 + cmd/gateway_delegate_test.go | 109 + cmd/gateway_lifecycle.go | 107 + cmd/gateway_managed.go | 38 +- cmd/gateway_subagent_announce_key_test.go | 47 + cmd/gateway_subagent_announce_queue.go | 50 +- cmd/gateway_subagent_announce_queue_test.go | 89 + cmd/gateway_subagent_recovery_test.go | 65 + docs/00-architecture-overview.md | 4 +- docs/01-agent-loop.md | 6 +- docs/03-tools-system.md | 57 +- docs/09-security.md | 12 +- docs/11-agent-teams.md | 58 +- docs/14-skills-runtime.md | 13 +- .../agent/delegation_artifact_workspace.go | 141 ++ internal/agent/loop_context.go | 72 +- .../agent/loop_context_collaboration_test.go | 235 ++ internal/agent/loop_finalize.go | 14 +- internal/agent/loop_history.go | 8 + internal/agent/loop_input_media.go | 113 +- internal/agent/loop_media.go | 79 +- internal/agent/loop_media_test.go | 99 +- internal/agent/loop_pipeline_adapter.go | 4 +- internal/agent/loop_pipeline_callbacks.go | 37 +- .../agent/loop_pipeline_callbacks_test.go | 97 + .../agent/loop_pipeline_tool_callbacks.go | 2 +- .../loop_pipeline_tool_callbacks_test.go | 40 + internal/agent/loop_run.go | 12 +- internal/agent/loop_tools.go | 13 +- internal/agent/loop_tracing.go | 12 +- internal/agent/loop_types.go | 18 +- internal/agent/media.go | 209 +- internal/agent/media_test.go | 247 +- .../prompt_builder_orchestration_test.go | 6 + internal/agent/prompt_config_types.go | 2 - internal/agent/systemprompt.go | 4 +- internal/agent/systemprompt_sections.go | 2 + .../channels/telegram/commands_subagents.go | 56 +- internal/childrun/admission.go | 901 ++++++++ internal/childrun/admission_test.go | 859 +++++++ internal/config/config.go | 4 +- internal/edition/edition.go | 18 +- internal/edition/edition_test.go | 9 + internal/gateway/bridge_context_test.go | 82 + internal/gateway/server.go | 51 +- internal/hooks/tracing.go | 2 +- internal/mcp/bridge_server.go | 6 +- internal/mcp/bridge_server_test.go | 20 + internal/mcp/crud_agent_links.go | 4 +- .../mcp/crud_agent_links_contract_test.go | 38 + internal/media/store.go | 6 + internal/orchestration/child_run_admission.go | 33 + internal/providers/claude_cli.go | 5 + internal/providers/claude_cli_mcp.go | 61 +- internal/providers/claude_cli_mcp_test.go | 66 + internal/providers/claude_cli_session.go | 18 +- internal/sandbox/docker.go | 99 +- internal/sandbox/docker_cache_key_test.go | 39 +- internal/sandbox/docker_mounts.go | 186 ++ internal/sandbox/docker_mounts_test.go | 342 +++ internal/sandbox/docker_resolve.go | 15 +- internal/sandbox/sandbox.go | 57 +- internal/store/agent_link_store.go | 20 +- internal/store/pg/agent_links.go | 10 +- internal/store/pg/factory.go | 4 +- internal/store/pg/subagent_tasks.go | 162 +- internal/store/pg/subagent_tasks_test.go | 472 ++++ internal/store/sqlitestore/agent-links.go | 11 +- .../store/sqlitestore/agent-links_test.go | 224 ++ internal/store/sqlitestore/factory.go | 4 +- internal/store/sqlitestore/schema.go | 92 +- internal/store/sqlitestore/schema.sql | 34 + internal/store/sqlitestore/subagent-tasks.go | 164 +- .../store/sqlitestore/subagent-tasks_test.go | 586 +++++ internal/store/stores.go | 1 + internal/store/subagent_store.go | 60 +- internal/tools/announce_queue.go | 121 +- .../tools/announce_queue_lifecycle_test.go | 50 + internal/tools/async_completion_delivery.go | 93 + .../tools/async_completion_ledger_test.go | 337 +++ internal/tools/async_completion_media.go | 80 + internal/tools/context_keys.go | 121 +- internal/tools/create_image.go | 40 +- .../tools/create_image_native_path_test.go | 53 + internal/tools/credentialed_exec.go | 2 +- internal/tools/delegate_completion_ledger.go | 238 ++ internal/tools/delegate_tool.go | 1053 ++++++++- internal/tools/delegate_tool_hooks_test.go | 2027 ++++++++++++++++- internal/tools/delegation_artifact_janitor.go | 708 ++++++ internal/tools/delegation_artifact_policy.go | 118 + .../tools/delegation_artifact_policy_test.go | 217 ++ .../delegation_artifact_result_policy.go | 36 + internal/tools/delegation_artifacts.go | 323 +++ internal/tools/delegation_artifacts_copy.go | 136 ++ internal/tools/delegation_artifacts_darwin.go | 32 + .../tools/delegation_artifacts_exchange.go | 193 ++ ...delegation_artifacts_hostile_posix_test.go | 37 + .../tools/delegation_artifacts_lifecycle.go | 195 ++ internal/tools/delegation_artifacts_linux.go | 32 + internal/tools/delegation_artifacts_posix.go | 390 ++++ .../tools/delegation_artifacts_publish.go | 475 ++++ internal/tools/delegation_artifacts_secure.go | 48 + internal/tools/delegation_artifacts_test.go | 751 ++++++ .../tools/delegation_artifacts_unsupported.go | 63 + .../tools/delegation_artifacts_windows.go | 574 +++++ .../tools/delegation_child_run_mode_test.go | 80 + internal/tools/edit.go | 12 +- internal/tools/filesystem.go | 77 +- internal/tools/filesystem_list.go | 29 +- internal/tools/filesystem_write.go | 14 +- internal/tools/message.go | 28 +- internal/tools/message_test.go | 66 +- internal/tools/read_audio.go | 25 +- internal/tools/read_audio_resolve.go | 27 +- internal/tools/read_audio_resolve_test.go | 216 ++ internal/tools/read_document.go | 55 +- internal/tools/read_document_resolve.go | 67 +- internal/tools/read_document_resolve_test.go | 126 +- internal/tools/read_image.go | 61 +- internal/tools/read_image_test.go | 19 + internal/tools/read_video.go | 34 +- internal/tools/read_video_resolve.go | 11 + internal/tools/sandbox_mount_test.go | 131 +- internal/tools/sandbox_utils.go | 86 + internal/tools/send_file.go | 3 + internal/tools/shell.go | 17 +- .../tools/shell_delegation_boundary_test.go | 189 ++ internal/tools/subagent.go | 94 +- .../subagent_announce_backpressure_test.go | 55 + internal/tools/subagent_announce_key_test.go | 47 + internal/tools/subagent_config.go | 20 +- .../tools/subagent_context_window_test.go | 7 +- internal/tools/subagent_control.go | 336 ++- internal/tools/subagent_exec.go | 159 +- internal/tools/subagent_persist.go | 158 +- internal/tools/subagent_roster.go | 22 +- internal/tools/subagent_roster_test.go | 64 + internal/tools/subagent_spawn.go | 497 ++-- internal/tools/subagent_spawn_test.go | 279 ++- internal/tools/subagent_spawn_tool.go | 96 +- internal/tools/subagent_spawn_tool_actions.go | 42 +- .../tools/subagent_task_lifecycle_test.go | 552 +++++ internal/tools/subagent_tracing.go | 12 +- internal/tools/team_metadata_keys.go | 71 +- internal/tracing/redaction.go | 114 + internal/tracing/redaction_test.go | 47 + internal/upgrade/version.go | 2 +- internal/workspace/resolver_impl.go | 30 +- internal/workspace/resolver_impl_test.go | 49 - internal/workspace/workspace_context.go | 21 +- ...6_subagent_tasks_root_agent_scope.down.sql | 9 + ...096_subagent_tasks_root_agent_scope.up.sql | 63 + skills/workspace-organizing/SKILL.md | 13 +- tests/integration/sqlite_smoke_test.go | 3 +- .../integration/v3_agent_links_store_test.go | 102 +- .../components/agents/subagents-section.tsx | 8 +- .../frontend/src/i18n/locales/en/agents.json | 1 + .../frontend/src/i18n/locales/ru/agents.json | 1 + .../frontend/src/i18n/locales/vi/agents.json | 1 + .../frontend/src/i18n/locales/zh/agents.json | 1 + ui/web/src/i18n/locales/en/agents.json | 1 + ui/web/src/i18n/locales/en/config.json | 2 +- ui/web/src/i18n/locales/en/teams.json | 1 + ui/web/src/i18n/locales/ko/agents.json | 1 + ui/web/src/i18n/locales/ko/config.json | 2 +- ui/web/src/i18n/locales/ko/teams.json | 1 + ui/web/src/i18n/locales/ru/agents.json | 1 + ui/web/src/i18n/locales/ru/config.json | 2 +- ui/web/src/i18n/locales/ru/teams.json | 1 + ui/web/src/i18n/locales/vi/agents.json | 1 + ui/web/src/i18n/locales/vi/config.json | 2 +- ui/web/src/i18n/locales/vi/teams.json | 1 + ui/web/src/i18n/locales/zh/agents.json | 1 + ui/web/src/i18n/locales/zh/config.json | 2 +- ui/web/src/i18n/locales/zh/teams.json | 1 + .../config-sections/subagents-section.tsx | 2 +- .../pages/config/hooks/use-config-defaults.ts | 2 +- .../pages/teams/links/link-create-dialog.tsx | 1 + 186 files changed, 18858 insertions(+), 1233 deletions(-) create mode 100644 cmd/gateway_child_run_lifecycle_test.go create mode 100644 cmd/gateway_consumer_outbound.go create mode 100644 cmd/gateway_delegate.go create mode 100644 cmd/gateway_delegate_test.go create mode 100644 cmd/gateway_subagent_announce_key_test.go create mode 100644 cmd/gateway_subagent_announce_queue_test.go create mode 100644 cmd/gateway_subagent_recovery_test.go create mode 100644 internal/agent/delegation_artifact_workspace.go create mode 100644 internal/agent/loop_context_collaboration_test.go create mode 100644 internal/childrun/admission.go create mode 100644 internal/childrun/admission_test.go create mode 100644 internal/mcp/crud_agent_links_contract_test.go create mode 100644 internal/orchestration/child_run_admission.go create mode 100644 internal/sandbox/docker_mounts.go create mode 100644 internal/sandbox/docker_mounts_test.go create mode 100644 internal/store/pg/subagent_tasks_test.go create mode 100644 internal/store/sqlitestore/agent-links_test.go create mode 100644 internal/store/sqlitestore/subagent-tasks_test.go create mode 100644 internal/tools/announce_queue_lifecycle_test.go create mode 100644 internal/tools/async_completion_delivery.go create mode 100644 internal/tools/async_completion_ledger_test.go create mode 100644 internal/tools/async_completion_media.go create mode 100644 internal/tools/delegate_completion_ledger.go create mode 100644 internal/tools/delegation_artifact_janitor.go create mode 100644 internal/tools/delegation_artifact_policy.go create mode 100644 internal/tools/delegation_artifact_policy_test.go create mode 100644 internal/tools/delegation_artifact_result_policy.go create mode 100644 internal/tools/delegation_artifacts.go create mode 100644 internal/tools/delegation_artifacts_copy.go create mode 100644 internal/tools/delegation_artifacts_darwin.go create mode 100644 internal/tools/delegation_artifacts_exchange.go create mode 100644 internal/tools/delegation_artifacts_hostile_posix_test.go create mode 100644 internal/tools/delegation_artifacts_lifecycle.go create mode 100644 internal/tools/delegation_artifacts_linux.go create mode 100644 internal/tools/delegation_artifacts_posix.go create mode 100644 internal/tools/delegation_artifacts_publish.go create mode 100644 internal/tools/delegation_artifacts_secure.go create mode 100644 internal/tools/delegation_artifacts_test.go create mode 100644 internal/tools/delegation_artifacts_unsupported.go create mode 100644 internal/tools/delegation_artifacts_windows.go create mode 100644 internal/tools/delegation_child_run_mode_test.go create mode 100644 internal/tools/shell_delegation_boundary_test.go create mode 100644 internal/tools/subagent_announce_backpressure_test.go create mode 100644 internal/tools/subagent_announce_key_test.go create mode 100644 internal/tools/subagent_roster_test.go create mode 100644 internal/tools/subagent_task_lifecycle_test.go create mode 100644 internal/tracing/redaction.go create mode 100644 internal/tracing/redaction_test.go create mode 100644 migrations/000096_subagent_tasks_root_agent_scope.down.sql create mode 100644 migrations/000096_subagent_tasks_root_agent_scope.up.sql diff --git a/.gitignore b/.gitignore index aaa1b579..784eb139 100644 --- a/.gitignore +++ b/.gitignore @@ -112,6 +112,7 @@ goclaw-patched-linux-amd64 ui/web/nginx.staging.conf # Local scratchpad / one-shot artifacts (never commit) +/.artifacts/ **/debug-*.log /*.sql scripts/docker-patch-*.sql diff --git a/README.md b/README.md index 442948d8..49a7e589 100644 --- a/README.md +++ b/README.md @@ -252,7 +252,11 @@ Open **About** dialog → click **Update Now** (admin only). The update includes Agent Orchestration

-Each agent runs with its own identity, tools, LLM provider, and context files. Three delegation modes — sync (wait), async (fire-and-forget), bidirectional — connected through explicit permission links with concurrency limits. +Each agent runs with its own identity, tools, LLM provider, and context files. +Agent Links define outbound, inbound, or bidirectional permission edges. +Delegation can run synchronously or asynchronously and exchanges files through +an isolated delegation workspace; validated outputs are published back under +the caller's `.delegations//` directory. > Details: [Agent Teams docs](https://docs.goclaw.sh/#teams-what-are-teams) diff --git a/cmd/gateway.go b/cmd/gateway.go index b0765269..0205d815 100644 --- a/cmd/gateway.go +++ b/cmd/gateway.go @@ -43,6 +43,7 @@ import ( mcpbridge "github.com/nextlevelbuilder/goclaw/internal/mcp" mcpoauth "github.com/nextlevelbuilder/goclaw/internal/mcp/oauth" "github.com/nextlevelbuilder/goclaw/internal/media" + "github.com/nextlevelbuilder/goclaw/internal/orchestration" "github.com/nextlevelbuilder/goclaw/internal/providers" "github.com/nextlevelbuilder/goclaw/internal/scheduler" "github.com/nextlevelbuilder/goclaw/internal/security" @@ -98,6 +99,29 @@ type snapshotBucketRefresher interface { RefreshBuckets(context.Context, []time.Time) (int, error) } +func recoverInterruptedSubagentTasks( + ctx context.Context, + stores *store.Stores, + retryDelay time.Duration, +) (int64, error) { + recoveryCtx := store.WithTenantID(ctx, store.MasterTenantID) + for { + recovered, err := stores.SubagentTaskRecovery.RecoverInterrupted(recoveryCtx) + if err == nil { + return recovered, nil + } + slog.Warn("subagent_tasks.recover_interrupted_retrying", "err", err) + + timer := time.NewTimer(retryDelay) + select { + case <-ctx.Done(): + timer.Stop() + return 0, ctx.Err() + case <-timer.C: + } + } +} + func backfillTraceCostsAfterPricingSync(ctx context.Context, stores *store.Stores, snapshots snapshotBucketRefresher) { if stores == nil { return @@ -292,6 +316,24 @@ func runGateway() { } } + // Accepted async child runs are process-owned and cannot resume after a + // restart. Reconcile their durable rows before wiring tools or accepting + // traffic so completion lookups never remain queued/running forever. + if pgStores.SubagentTaskRecovery != nil { + startupCtx, stopStartup := signal.NotifyContext( + context.Background(), syscall.SIGINT, syscall.SIGTERM, + ) + n, err := recoverInterruptedSubagentTasks(startupCtx, pgStores, time.Second) + stopStartup() + if err != nil { + slog.Info("subagent_tasks.recover_interrupted_aborted", "err", err) + return + } + if n > 0 { + slog.Info("subagent_tasks.recover_interrupted", "count", n) + } + } + if traceCollector != nil { defer traceCollector.Stop() // OTel OTLP export: compiled via build tags. Build with 'go build -tags otel' to enable. @@ -456,7 +498,8 @@ func runGateway() { } // Subagent system (secureCLI store wired so subagent ExecTools enforce the gate) - subagentMgr := setupSubagents(providerRegistry, cfg, msgBus, toolsReg, workspace, sandboxMgr, pgStores.SecureCLI, usageCapSvc) + childRunAdmission := orchestration.NewChildRunAdmission(edition.Current().ChildRunLimit(), 128) + subagentMgr := setupSubagents(providerRegistry, cfg, msgBus, toolsReg, workspace, sandboxMgr, pgStores.SecureCLI, usageCapSvc, childRunAdmission) if subagentMgr != nil { // Wire announce queue for batched subagent result delivery (matching TS debounce pattern). announceQueue := tools.NewAnnounceQueue(1000, 20, makeDelegateAnnounceCallback(subagentMgr, msgBus)) @@ -577,7 +620,7 @@ func runGateway() { var mcpPool *mcpbridge.Pool var mediaStore *media.Store var postTurn tools.PostTurnProcessor - contextFileInterceptor, mcpPool, mediaStore, postTurn = wireExtras(pgStores, agentRouter, providerRegistry, modelReg, msgBus, pgStores.Sessions, toolsReg, toolPE, skillsLoader, hasMemory, traceCollector, workspace, cfg.Gateway.InjectionAction, cfg, sandboxMgr, redisClient, domainBus, usageCapSvc, mcpOAuthRefresher) + contextFileInterceptor, mcpPool, mediaStore, postTurn = wireExtras(pgStores, agentRouter, providerRegistry, modelReg, msgBus, pgStores.Sessions, toolsReg, toolPE, skillsLoader, hasMemory, traceCollector, workspace, cfg.Gateway.InjectionAction, cfg, sandboxMgr, redisClient, domainBus, usageCapSvc, mcpOAuthRefresher, childRunAdmission) if mcpPool != nil { defer mcpPool.Stop() } @@ -1042,9 +1085,11 @@ func runGateway() { sandboxMgr: sandboxMgr, postTurn: postTurn, subagentMgr: subagentMgr, + childRunAdmission: childRunAdmission, consumerTeamStore: consumerTeamStore, auditCh: auditCh, sigCh: sigCh, + terminateProcess: os.Exit, }) } diff --git a/cmd/gateway_agents.go b/cmd/gateway_agents.go index 27703d20..4a8efc30 100644 --- a/cmd/gateway_agents.go +++ b/cmd/gateway_agents.go @@ -11,6 +11,7 @@ import ( "github.com/nextlevelbuilder/goclaw/internal/bus" "github.com/nextlevelbuilder/goclaw/internal/config" "github.com/nextlevelbuilder/goclaw/internal/memory" + "github.com/nextlevelbuilder/goclaw/internal/orchestration" "github.com/nextlevelbuilder/goclaw/internal/providers" "github.com/nextlevelbuilder/goclaw/internal/sandbox" "github.com/nextlevelbuilder/goclaw/internal/store" @@ -163,7 +164,7 @@ func buildEmbeddingProvider( return nil } -func setupSubagents(providerReg *providers.Registry, cfg *config.Config, msgBus *bus.MessageBus, toolsReg *tools.Registry, workspace string, sandboxMgr sandbox.Manager, secureCLIStore store.SecureCLIStore, usageCapSvc *usagecaps.Service) *tools.SubagentManager { +func setupSubagents(providerReg *providers.Registry, cfg *config.Config, msgBus *bus.MessageBus, toolsReg *tools.Registry, workspace string, sandboxMgr sandbox.Manager, secureCLIStore store.SecureCLIStore, usageCapSvc *usagecaps.Service, admission *orchestration.ChildRunAdmission) *tools.SubagentManager { names := providerReg.List(context.Background()) if len(names) == 0 { return nil @@ -211,7 +212,7 @@ func setupSubagents(providerReg *providers.Registry, cfg *config.Config, msgBus return reg } - manager := tools.NewSubagentManager(provider, providerReg, agentCfg.Model, msgBus, toolsFactory, subCfg) + manager := tools.NewSubagentManagerWithAdmission(provider, providerReg, agentCfg.Model, msgBus, toolsFactory, subCfg, admission) manager.SetUsageCapService(usageCapSvc) manager.SetAgentBudget(agentCfg.ContextWindow, agentCfg.MaxTokens) return manager diff --git a/cmd/gateway_child_run_lifecycle_test.go b/cmd/gateway_child_run_lifecycle_test.go new file mode 100644 index 00000000..6b87f93e --- /dev/null +++ b/cmd/gateway_child_run_lifecycle_test.go @@ -0,0 +1,52 @@ +package cmd + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/google/uuid" + + "github.com/nextlevelbuilder/goclaw/internal/orchestration" + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +func TestDrainChildRunsWithRetryReturnsTypedFailure(t *testing.T) { + admission := orchestration.NewChildRunAdmission(1, 2) + started := make(chan struct{}) + release := make(chan struct{}) + ticket, err := admission.Enqueue(context.Background(), orchestration.ChildRunConstraints{ + TenantID: store.MasterTenantID, + TaskID: uuid.NewString(), + }, func(context.Context, *orchestration.ChildRunLease) { + close(started) + <-release + }) + if err != nil { + t.Fatal(err) + } + if err := ticket.Activate(); err != nil { + t.Fatal(err) + } + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("child run did not start") + } + + err = drainChildRunsWithRetry(admission, 10*time.Millisecond, 10*time.Millisecond) + if !errors.Is(err, orchestration.ErrChildRunDrainTimeout) { + t.Fatalf("drain error = %v, want typed timeout", err) + } + + close(release) + select { + case <-ticket.Done(): + case <-time.After(time.Second): + t.Fatal("child run did not finish after forced termination signal") + } + if err := admission.Close(context.Background()); err != nil { + t.Fatalf("final drain: %v", err) + } +} diff --git a/cmd/gateway_consumer_handlers.go b/cmd/gateway_consumer_handlers.go index 9588eabf..cb80c848 100644 --- a/cmd/gateway_consumer_handlers.go +++ b/cmd/gateway_consumer_handlers.go @@ -79,6 +79,7 @@ func handleSubagentAnnounce( if sid := msg.Metadata[tools.MetaOriginRootSpanID]; sid != "" { parentRootSpanID, _ = uuid.Parse(sid) } + rootAgentID, _ := uuid.Parse(msg.Metadata[tools.MetaSubagentRootAgentID]) // Group-scoped UserID for subagent announce (same logic as main lane). announceUserID := msg.UserID @@ -116,9 +117,7 @@ func handleSubagentAnnounce( originSenderID := msg.Metadata[tools.MetaOriginSenderID] originRole := msg.Metadata[tools.MetaOriginRole] - queueKey := fmt.Sprintf("%s:%s", msg.TenantID, sessionKey) routing := subagentAnnounceRouting{ - QueueKey: queueKey, SessionKey: sessionKey, TenantID: msg.TenantID, OrigChannel: origChannel, @@ -130,10 +129,16 @@ func handleSubagentAnnounce( SenderID: originSenderID, Role: originRole, ParentAgent: parentAgent, + RootAgentID: rootAgentID, ParentTraceID: parentTraceID, ParentRootSpanID: parentRootSpanID, OutMeta: buildAnnounceOutMeta(origLocalKey), } + // Batch only announces with identical routing and authority. A session can + // be shared by multiple group senders or local topic keys; using only + // tenant+session would let the first item's sender/role govern the rest. + queueKey := subagentAnnounceRoutingKey(routing) + routing.QueueKey = queueKey // Enqueue into producer-consumer queue using tenant-scoped key from routing. isProcessor := enqueueSubagentAnnounce(queueKey, entry) @@ -142,7 +147,9 @@ func handleSubagentAnnounce( defer safego.Recover(nil, "component", "subagent_announce_loop", "session", sessionKey) // Fetch live roster for merged announce context. - roster := deps.SubagentMgr.RosterForParent(parentAgent) + roster := deps.SubagentMgr.RosterForParent(tools.TaskScope{ + TenantID: msg.TenantID, RootAgentID: rootAgentID, RootAgentKey: parentAgent, + }) processSubagentAnnounceLoop(ctx, routing, roster, deps.SubagentMgr, deps.Sched, deps.MsgBus, deps.Cfg, deps.ChannelMgr) }) @@ -151,6 +158,22 @@ func handleSubagentAnnounce( return true } +func subagentAnnounceRoutingKey(r subagentAnnounceRouting) string { + return strings.Join([]string{ + r.TenantID.String(), + r.RootAgentID.String(), + r.ParentAgent, + r.SessionKey, + r.OrigChannel, + r.OrigChatID, + r.OrigPeerKind, + r.OrigLocalKey, + r.UserID, + r.SenderID, + r.Role, + }, "\x00") +} + // handleTeammateMessage processes teammate messages: bypass debounce, route to target // agent session using the "team" lane, then announce result back to lead. // Returns true if the message was handled (caller should continue). @@ -560,11 +583,14 @@ func buildTeammateAnnounce(ctx context.Context, outcome scheduler.RunOutcome, se } else if outcome.Result == nil { slog.Warn("teammate message: nil result without error", "from", senderID) return "", nil, false - } else if (outcome.Result.Content == "" && len(outcome.Result.Media) == 0) || agent.IsSilentReply(outcome.Result.Content) { + } else if normalized, shouldDeliver := normalizeAgentOutboundContent( + outcome.Result.Content, + len(outcome.Result.Media), + ); !shouldDeliver { slog.Info("teammate message: suppressed silent/empty reply", "from", senderID) return "", nil, false } else { - content = outcome.Result.Content + content = normalized media = outcome.Result.Media } diff --git a/cmd/gateway_consumer_normal.go b/cmd/gateway_consumer_normal.go index e88328fd..df2ab9c6 100644 --- a/cmd/gateway_consumer_normal.go +++ b/cmd/gateway_consumer_normal.go @@ -575,9 +575,14 @@ func processNormalMessage( return } - // Suppress empty/NO_REPLY responses (matching TS normalize-reply.ts). - // Still publish an empty outbound so channels can clean up placeholder/thinking indicators. - if outcome.Result.Content == "" || agent.IsSilentReply(outcome.Result.Content) { + // Suppress silent text only when the result has no media. A tool or + // delegate may legitimately return NO_REPLY with an attached artifact; + // that must continue as a media-only outbound message. + resultContent, shouldDeliver := normalizeAgentOutboundContent( + outcome.Result.Content, + len(outcome.Result.Media), + ) + if !shouldDeliver { slog.Info("inbound: suppressed silent/empty reply", "channel", channel, "chat_id", chatID, @@ -615,7 +620,7 @@ func processNormalMessage( // Sanitize voice agent replies: replace technical errors with user-friendly fallback. replyContent := voiceguard.SanitizeReply( deps.Cfg.Channels.Telegram.VoiceAgentID, agentKey, - channel, peerKind, inboundContent, outcome.Result.Content, + channel, peerKind, inboundContent, resultContent, deps.Cfg.Channels.Telegram.AudioGuardFallbackTranscript, deps.Cfg.Channels.Telegram.AudioGuardFallbackNoTranscript, deps.Cfg.Channels.Telegram.AudioGuardErrorMarkers, diff --git a/cmd/gateway_consumer_outbound.go b/cmd/gateway_consumer_outbound.go new file mode 100644 index 00000000..2bbe2909 --- /dev/null +++ b/cmd/gateway_consumer_outbound.go @@ -0,0 +1,16 @@ +package cmd + +import "github.com/nextlevelbuilder/goclaw/internal/agent" + +// normalizeAgentOutboundContent keeps silent text out of user-facing messages +// while allowing attached media to continue through channel delivery. +func normalizeAgentOutboundContent(content string, mediaCount int) (string, bool) { + isSilent := content == "" || agent.IsSilentReply(content) + if !isSilent { + return content, true + } + if mediaCount == 0 { + return "", false + } + return "", true +} diff --git a/cmd/gateway_delegate.go b/cmd/gateway_delegate.go new file mode 100644 index 00000000..bb726062 --- /dev/null +++ b/cmd/gateway_delegate.go @@ -0,0 +1,54 @@ +package cmd + +import ( + "context" + "fmt" + "time" + + "github.com/google/uuid" + + "github.com/nextlevelbuilder/goclaw/internal/agent" + "github.com/nextlevelbuilder/goclaw/internal/bus" + "github.com/nextlevelbuilder/goclaw/internal/sandbox" + "github.com/nextlevelbuilder/goclaw/internal/tools" +) + +// buildAgentLinkRunRequest preserves the origin's authorization-bearing +// identity while keeping delegation on its internal delivery channel. +func buildAgentLinkRunRequest(req tools.DelegateRequest, sessionKey string) agent.RunRequest { + return agent.RunRequest{ + RunID: uuid.New().String(), + SessionKey: sessionKey, + Message: req.Task, + UserID: req.UserID, + SenderID: req.SenderID, + Role: req.Role, + Channel: "delegate", + ChannelType: req.ChannelType, + ChatID: req.ChatID, + PeerKind: req.PeerKind, + RunKind: "delegate", + DelegationID: req.DelegationID, + ParentAgentID: req.FromAgentKey, + WorkspaceChannel: req.Channel, + WorkspaceChatID: req.ChatID, + DelegateInputsPath: req.DelegateInputsPath, + DelegateOutputsPath: req.DelegateOutputsPath, + } +} + +// Agent Link artifacts are published from the exchange manifest. Raw delegate +// media paths point into B's ephemeral workspace and must never cross back to A. +func agentMediaToBusFiles(_ []agent.MediaResult) []bus.MediaFile { return nil } + +func releaseDelegationSandbox(ctx context.Context, manager sandbox.Manager, sessionKey string) error { + if manager == nil { + return nil + } + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + if err := manager.Release(cleanupCtx, sessionKey); err != nil { + return fmt.Errorf("delegation sandbox release failed") + } + return nil +} diff --git a/cmd/gateway_delegate_test.go b/cmd/gateway_delegate_test.go new file mode 100644 index 00000000..0bc9ddd0 --- /dev/null +++ b/cmd/gateway_delegate_test.go @@ -0,0 +1,109 @@ +package cmd + +import ( + "context" + "errors" + "path/filepath" + "testing" + + "github.com/google/uuid" + + "github.com/nextlevelbuilder/goclaw/internal/agent" + "github.com/nextlevelbuilder/goclaw/internal/sandbox" + "github.com/nextlevelbuilder/goclaw/internal/tools" +) + +type delegationReleaseManager struct { + released string + err error +} + +func (m *delegationReleaseManager) Get(context.Context, string, string, *sandbox.Config, ...sandbox.GetOption) (sandbox.Sandbox, error) { + return nil, errors.New("not implemented") +} +func (m *delegationReleaseManager) Release(_ context.Context, key string) error { + m.released = key + return m.err +} +func (*delegationReleaseManager) ReleaseAll(context.Context) error { return nil } +func (*delegationReleaseManager) Stop() {} +func (*delegationReleaseManager) Stats() map[string]any { return nil } + +func TestBuildAgentLinkRunRequestPreservesGroupAuthorizationScope(t *testing.T) { + req := tools.DelegateRequest{ + FromAgentKey: "coordinator", + Task: "create the report", + DelegationID: uuid.NewString(), + UserID: "group:telegram:-100123", + SenderID: "386246614", + Role: "viewer", + Channel: "telegram-main", + ChannelType: "telegram", + ChatID: "-100123", + PeerKind: "group", + DelegateInputsPath: filepath.Join(t.TempDir(), "inputs"), + DelegateOutputsPath: filepath.Join(t.TempDir(), "outputs"), + } + + got := buildAgentLinkRunRequest(req, "delegate:session") + + if got.UserID != req.UserID || got.SenderID != req.SenderID || got.Role != req.Role { + t.Fatalf("authorization scope = (%q, %q, %q), want (%q, %q, %q)", + got.UserID, got.SenderID, got.Role, req.UserID, req.SenderID, req.Role) + } + if got.Channel != "delegate" || got.ChannelType != req.ChannelType { + t.Fatalf("channel = (%q, %q), want (delegate, %q)", got.Channel, got.ChannelType, req.ChannelType) + } + if got.ChatID != req.ChatID || got.PeerKind != req.PeerKind { + t.Fatalf("chat scope = (%q, %q), want (%q, %q)", + got.ChatID, got.PeerKind, req.ChatID, req.PeerKind) + } + if got.WorkspaceChannel != req.Channel || got.WorkspaceChatID != req.ChatID { + t.Fatalf("workspace scope = (%q, %q), want (%q, %q)", + got.WorkspaceChannel, got.WorkspaceChatID, req.Channel, req.ChatID) + } + if got.RunID == "" || got.RunKind != "delegate" || got.DelegationID != req.DelegationID { + t.Fatalf("delegate classification = %#v", got) + } + if got.DelegateInputsPath != req.DelegateInputsPath || + got.DelegateOutputsPath != req.DelegateOutputsPath { + t.Fatalf("artifact runtime wiring = (%q, %q), want (%q, %q)", + got.DelegateInputsPath, got.DelegateOutputsPath, + req.DelegateInputsPath, req.DelegateOutputsPath) + } + if got.Media != nil { + t.Fatalf("run media = %#v, want exchange-only input delivery", got.Media) + } +} + +func TestAgentMediaToBusFilesDiscardsEphemeralDelegateMedia(t *testing.T) { + got := agentMediaToBusFiles([]agent.MediaResult{{ + Path: filepath.Join(t.TempDir(), "ephemeral.png"), + ContentType: "image/png", + }}) + if got != nil { + t.Fatalf("media = %#v, want raw delegate media discarded", got) + } +} + +func TestReleaseDelegationSandboxUsesExactSessionKey(t *testing.T) { + manager := &delegationReleaseManager{} + const sessionKey = "delegate:from:target:4fe8220f-07f1-4e64-a95c-b49ebc39db4a" + if err := releaseDelegationSandbox(context.Background(), manager, sessionKey); err != nil { + t.Fatalf("releaseDelegationSandbox: %v", err) + } + if manager.released != sessionKey { + t.Fatalf("released key = %q, want %q", manager.released, sessionKey) + } +} + +func TestReleaseDelegationSandboxRedactsManagerFailure(t *testing.T) { + manager := &delegationReleaseManager{err: errors.New("/private/exchange/inputs remained mounted")} + err := releaseDelegationSandbox(context.Background(), manager, "delegate:session") + if err == nil { + t.Fatal("releaseDelegationSandbox unexpectedly succeeded") + } + if got := err.Error(); got != "delegation sandbox release failed" { + t.Fatalf("error = %q, want redacted release failure", got) + } +} diff --git a/cmd/gateway_lifecycle.go b/cmd/gateway_lifecycle.go index 1a3a37b8..7a87be0e 100644 --- a/cmd/gateway_lifecycle.go +++ b/cmd/gateway_lifecycle.go @@ -2,6 +2,7 @@ package cmd import ( "context" + "fmt" "log/slog" "os" "strings" @@ -14,6 +15,7 @@ import ( "github.com/nextlevelbuilder/goclaw/internal/config" "github.com/nextlevelbuilder/goclaw/internal/edition" "github.com/nextlevelbuilder/goclaw/internal/heartbeat" + "github.com/nextlevelbuilder/goclaw/internal/orchestration" "github.com/nextlevelbuilder/goclaw/internal/sandbox" "github.com/nextlevelbuilder/goclaw/internal/scheduler" "github.com/nextlevelbuilder/goclaw/internal/store" @@ -33,9 +35,71 @@ type lifecycleDeps struct { sandboxMgr sandbox.Manager postTurn tools.PostTurnProcessor subagentMgr *tools.SubagentManager + childRunAdmission *orchestration.ChildRunAdmission consumerTeamStore store.TeamStore auditCh chan bus.AuditEventPayload sigCh chan os.Signal + terminateProcess func(int) +} + +func drainChildRunsWithRetry( + admission *orchestration.ChildRunAdmission, + firstTimeout time.Duration, + retryTimeout time.Duration, +) error { + if admission == nil { + return nil + } + for attempt, timeout := range []time.Duration{firstTimeout, retryTimeout} { + drainCtx, drainCancel := context.WithTimeout(context.Background(), timeout) + err := admission.Close(drainCtx) + drainCancel() + if err == nil { + return nil + } + slog.Error("gateway: child-run drain attempt failed", + "attempt", attempt+1, "timeout", timeout, "error", err) + } + return fmt.Errorf("%w after retry", orchestration.ErrChildRunDrainTimeout) +} + +func drainSubagentManagerWithRetry( + manager *tools.SubagentManager, + firstTimeout time.Duration, + retryTimeout time.Duration, +) error { + if manager == nil { + return nil + } + for attempt, timeout := range []time.Duration{firstTimeout, retryTimeout} { + drainCtx, drainCancel := context.WithTimeout(context.Background(), timeout) + err := manager.CloseContext(drainCtx) + drainCancel() + if err == nil { + return nil + } + slog.Error("gateway: subagent lifecycle drain attempt failed", + "attempt", attempt+1, "timeout", timeout, "error", err) + } + return fmt.Errorf("%w after retry", tools.ErrSubagentLifecycleDrainTimeout) +} + +func drainDelegateToolWithRetry( + tool interface{ CloseContext(context.Context) error }, + firstTimeout time.Duration, + retryTimeout time.Duration, +) error { + for attempt, timeout := range []time.Duration{firstTimeout, retryTimeout} { + drainCtx, drainCancel := context.WithTimeout(context.Background(), timeout) + err := tool.CloseContext(drainCtx) + drainCancel() + if err == nil { + return nil + } + slog.Error("gateway: delegate completion drain attempt failed", + "attempt", attempt+1, "timeout", timeout, "error", err) + } + return fmt.Errorf("delegate completion drain failed after retry") } // runLifecycle wires config-reload subscribers, starts consumers, task recovery, @@ -198,6 +262,20 @@ func (d *gatewayDeps) runLifecycle( // Broadcast shutdown event d.server.BroadcastEvent(*protocol.NewEvent(protocol.EventShutdown, nil)) + // Close child-run intake first. A drain timeout must terminate without + // unwinding runGateway defers under a still-live child callback. + if deps.childRunAdmission != nil { + if err := drainChildRunsWithRetry(deps.childRunAdmission, 30*time.Second, 5*time.Second); err != nil { + slog.Error("gateway: terminating after child-run drain failure", "error", err) + terminate := deps.terminateProcess + if terminate == nil { + terminate = os.Exit + } + terminate(1) + return + } + } + // Stop channels, cron, heartbeat, and task ticker d.channelMgr.StopAll(context.Background()) d.pgStores.Cron.Stop() @@ -216,6 +294,35 @@ func (d *gatewayDeps) runLifecycle( close(deps.auditCh) } + if delegate, ok := d.toolsReg.Get("delegate"); ok { + if closer, ok := delegate.(interface { + CloseContext(context.Context) error + }); ok { + if err := drainDelegateToolWithRetry(closer, 65*time.Second, 10*time.Second); err != nil { + slog.Error("gateway: terminating after delegate completion drain failure", "error", err) + terminate := deps.terminateProcess + if terminate == nil { + terminate = os.Exit + } + terminate(1) + return + } + } else if closer, ok := delegate.(interface{ Close() }); ok { + closer.Close() + } + } + if deps.subagentMgr != nil { + if err := drainSubagentManagerWithRetry(deps.subagentMgr, 65*time.Second, 10*time.Second); err != nil { + slog.Error("gateway: terminating after subagent lifecycle drain failure", "error", err) + terminate := deps.terminateProcess + if terminate == nil { + terminate = os.Exit + } + terminate(1) + return + } + } + // Close provider resources (e.g. Claude CLI temp files) d.providerRegistry.Close() diff --git a/cmd/gateway_managed.go b/cmd/gateway_managed.go index 5e775b37..50f3914f 100644 --- a/cmd/gateway_managed.go +++ b/cmd/gateway_managed.go @@ -62,6 +62,7 @@ func wireExtras( domainBus eventbus.DomainEventBus, usageCapSvc *usagecaps.Service, mcpOAuthProvider mcpbridge.OAuthTokenProvider, // nil = OAuth injection disabled + childRunAdmission *orchestration.ChildRunAdmission, ) (*tools.ContextFileInterceptor, *mcpbridge.Pool, *media.Store, tools.PostTurnProcessor) { // 1. Build cache instances (in-memory or Redis depending on build tags) agentCtxCache, userCtxCache := makeCaches(redisClient) @@ -450,25 +451,32 @@ func wireExtras( // Link delegate trace to parent trace delegateCtx := tracing.WithDelegateParentTraceID(ctx, tracing.TraceIDFromContext(ctx)) - runReq := agent.RunRequest{ - RunID: uuid.New().String(), - SessionKey: sessionKey, - Message: req.Task, - UserID: req.UserID, - Channel: "delegate", - RunKind: "delegate", - DelegationID: req.DelegationID, - ParentAgentID: req.FromAgentKey, + runReq := buildAgentLinkRunRequest(req, sessionKey) + var delegateTraceID uuid.UUID + runReq.OnTraceCreated = func(traceID uuid.UUID) { + delegateTraceID = traceID + if req.OnTraceCreated != nil { + req.OnTraceCreated(traceID) + } } - result, err := loop.Run(delegateCtx, runReq) - if err != nil { - return tools.DelegateResult{}, err + result, runErr := loop.Run(delegateCtx, runReq) + if releaseErr := releaseDelegationSandbox(ctx, sandboxMgr, sessionKey); releaseErr != nil { + return tools.DelegateResult{TraceID: delegateTraceID}, releaseErr } - cr := orchestration.CaptureFromRunResult(result, 0) - return tools.DelegateResult{Content: cr.Content, Media: cr.Media}, nil + if runErr != nil { + return tools.DelegateResult{TraceID: delegateTraceID}, runErr + } + return tools.DelegateResult{ + Content: result.Content, + Media: agentMediaToBusFiles(result.Media), + TraceID: delegateTraceID, + }, nil } - delegateTool := tools.NewDelegateTool(stores.AgentLinks, stores.Agents, domainBus, delegateRunFn) + delegateTool := tools.NewDelegateToolWithAdmission(stores.AgentLinks, stores.Agents, domainBus, delegateRunFn, childRunAdmission) + delegateTool.SetDataDir(appCfg.DataDir) + delegateTool.SetWorkspace(workspace) delegateTool.SetMsgBus(msgBus) + delegateTool.SetTaskStore(stores.SubagentTasks) delegateTool.SetHookDispatcher(hookDispatcher) toolsReg.Register(delegateTool) slog.Info("delegate tool wired") diff --git a/cmd/gateway_subagent_announce_key_test.go b/cmd/gateway_subagent_announce_key_test.go new file mode 100644 index 00000000..80d33bef --- /dev/null +++ b/cmd/gateway_subagent_announce_key_test.go @@ -0,0 +1,47 @@ +package cmd + +import ( + "testing" + + "github.com/google/uuid" +) + +func TestSubagentAnnounceRoutingKeyScopesRoutingAndAuthority(t *testing.T) { + base := subagentAnnounceRouting{ + TenantID: uuid.New(), + RootAgentID: uuid.New(), + ParentAgent: "root", + SessionKey: "session", + OrigChannel: "telegram", + OrigChatID: "chat", + OrigPeerKind: "group", + OrigLocalKey: "topic", + UserID: "user", + SenderID: "sender", + Role: "operator", + } + want := subagentAnnounceRoutingKey(base) + + cases := map[string]func(*subagentAnnounceRouting){ + "tenant": func(v *subagentAnnounceRouting) { v.TenantID = uuid.New() }, + "root id": func(v *subagentAnnounceRouting) { v.RootAgentID = uuid.New() }, + "parent agent": func(v *subagentAnnounceRouting) { v.ParentAgent += "-other" }, + "session": func(v *subagentAnnounceRouting) { v.SessionKey += "-other" }, + "channel": func(v *subagentAnnounceRouting) { v.OrigChannel += "-other" }, + "chat": func(v *subagentAnnounceRouting) { v.OrigChatID += "-other" }, + "peer kind": func(v *subagentAnnounceRouting) { v.OrigPeerKind += "-other" }, + "local key": func(v *subagentAnnounceRouting) { v.OrigLocalKey += "-other" }, + "user": func(v *subagentAnnounceRouting) { v.UserID += "-other" }, + "sender": func(v *subagentAnnounceRouting) { v.SenderID += "-other" }, + "role": func(v *subagentAnnounceRouting) { v.Role += "-other" }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + got := base + mutate(&got) + if key := subagentAnnounceRoutingKey(got); key == want { + t.Fatalf("routing key did not change when %s changed", name) + } + }) + } +} diff --git a/cmd/gateway_subagent_announce_queue.go b/cmd/gateway_subagent_announce_queue.go index 1e20cf43..6085bafa 100644 --- a/cmd/gateway_subagent_announce_queue.go +++ b/cmd/gateway_subagent_announce_queue.go @@ -27,7 +27,9 @@ func makeDelegateAnnounceCallback( msgBus *bus.MessageBus, ) func(sessionKey string, items []tools.AnnounceQueueItem, meta tools.AnnounceMetadata) { return func(sessionKey string, items []tools.AnnounceQueueItem, meta tools.AnnounceMetadata) { - roster := subagentMgr.RosterForParent(meta.ParentAgent) + roster := subagentMgr.RosterForParent(tools.TaskScope{ + TenantID: meta.OriginTenantID, RootAgentID: meta.RootAgentID, RootAgentKey: meta.ParentAgent, + }) content := tools.FormatBatchedAnnounce(items, roster) senderID := fmt.Sprintf("subagent:batch-%d", len(items)) label := items[0].Label @@ -35,12 +37,13 @@ func makeDelegateAnnounceCallback( label = fmt.Sprintf("%d tasks", len(items)) } batchMeta := map[string]string{ - tools.MetaOriginChannel: meta.OriginChannel, - tools.MetaOriginPeerKind: meta.OriginPeerKind, - tools.MetaParentAgent: meta.ParentAgent, - tools.MetaSubagentLabel: label, - tools.MetaOriginTraceID: meta.OriginTraceID, - tools.MetaOriginRootSpanID: meta.OriginRootSpanID, + tools.MetaOriginChannel: meta.OriginChannel, + tools.MetaOriginPeerKind: meta.OriginPeerKind, + tools.MetaParentAgent: meta.ParentAgent, + tools.MetaSubagentRootAgentID: meta.RootAgentID.String(), + tools.MetaSubagentLabel: label, + tools.MetaOriginTraceID: meta.OriginTraceID, + tools.MetaOriginRootSpanID: meta.OriginRootSpanID, } if meta.OriginLocalKey != "" { batchMeta[tools.MetaOriginLocalKey] = meta.OriginLocalKey @@ -69,7 +72,7 @@ func makeDelegateAnnounceCallback( "tasks": len(items), }) - msgBus.PublishInbound(bus.InboundMessage{ + delivered := tools.PublishAsyncCompletion(context.Background(), msgBus, bus.InboundMessage{ Channel: "system", SenderID: senderID, ChatID: meta.OriginChatID, @@ -79,6 +82,30 @@ func makeDelegateAnnounceCallback( Metadata: batchMeta, Media: batchMedia, }) + for _, item := range items { + if !item.DurablyPersisted { + slog.Error("subagent.batch_announce_without_durable_terminal", + "task_id", item.SubagentID, + "completion_id", item.CompletionID, + "root_agent_id", meta.RootAgentID, + "delivered", delivered, + ) + continue + } + subagentMgr.UpdateAnnouncementStatus( + store.WithTenantID(context.Background(), meta.OriginTenantID), + meta.RootAgentID, + item.CompletionID, + delivered, + ) + } + if !delivered { + slog.Warn("subagent.batch_announce_deferred_to_ledger", + "root_agent_id", meta.RootAgentID, + "batch_size", len(items), + "reason", "inbound_bus_full", + ) + } } } @@ -96,7 +123,7 @@ type subagentAnnounceEntry struct { // subagentAnnounceRouting holds shared routing info captured by the first enqueue. type subagentAnnounceRouting struct { - QueueKey string // tenant-scoped key for sync.Map (tenantID:sessionKey) + QueueKey string // tenant/root/session/topic/user/authority-scoped key SessionKey string // original session key (no tenant prefix) for RunRequest TenantID uuid.UUID // preserved for tenant-scoped scheduling OrigChannel string @@ -108,6 +135,7 @@ type subagentAnnounceRouting struct { SenderID string // real acting sender (preserves permission attribution through re-ingress, #915) Role string // caller's RBAC role; bypasses per-user grants for admin/operator/owner (#915) ParentAgent string + RootAgentID uuid.UUID ParentTraceID uuid.UUID ParentRootSpanID uuid.UUID OutMeta map[string]string @@ -156,7 +184,9 @@ func processSubagentAnnounceLoop( } // Refresh roster each iteration for up-to-date task statuses. - roster = subagentMgr.RosterForParent(r.ParentAgent) + roster = subagentMgr.RosterForParent(tools.TaskScope{ + TenantID: r.TenantID, RootAgentID: r.RootAgentID, RootAgentKey: r.ParentAgent, + }) content := buildMergedSubagentAnnounce(entries, roster) // Collect media from all entries. diff --git a/cmd/gateway_subagent_announce_queue_test.go b/cmd/gateway_subagent_announce_queue_test.go new file mode 100644 index 00000000..7d24eb44 --- /dev/null +++ b/cmd/gateway_subagent_announce_queue_test.go @@ -0,0 +1,89 @@ +package cmd + +import ( + "context" + "testing" + "time" + + "github.com/google/uuid" + + "github.com/nextlevelbuilder/goclaw/internal/bus" + "github.com/nextlevelbuilder/goclaw/internal/store" + "github.com/nextlevelbuilder/goclaw/internal/tools" +) + +type recordingGatewayTaskStore struct { + metadata chan map[string]any +} + +func (*recordingGatewayTaskStore) Create(context.Context, *store.SubagentTaskData) error { + return nil +} +func (*recordingGatewayTaskStore) Get(context.Context, uuid.UUID, uuid.UUID) (*store.SubagentTaskData, error) { + return nil, nil +} +func (*recordingGatewayTaskStore) UpdateStatus(context.Context, uuid.UUID, uuid.UUID, string, *string, int, int64, int64) error { + return nil +} +func (*recordingGatewayTaskStore) ListByParent(context.Context, uuid.UUID, string) ([]store.SubagentTaskData, error) { + return nil, nil +} +func (*recordingGatewayTaskStore) ListBySession(context.Context, uuid.UUID, string) ([]store.SubagentTaskData, error) { + return nil, nil +} +func (*recordingGatewayTaskStore) Archive(context.Context, uuid.UUID, time.Duration, int) (int64, error) { + return 0, nil +} +func (s *recordingGatewayTaskStore) UpdateMetadata(_ context.Context, _ uuid.UUID, _ uuid.UUID, metadata map[string]any) error { + s.metadata <- metadata + return nil +} + +func TestSubagentBatchAnnouncementDoesNotBlockOnFullInboundBus(t *testing.T) { + messageBus := bus.New() + for range 1000 { + messageBus.PublishInbound(bus.InboundMessage{Content: "fill"}) + } + manager := tools.NewSubagentManager(nil, nil, "", messageBus, nil, tools.SubagentConfig{}) + taskStore := &recordingGatewayTaskStore{metadata: make(chan map[string]any, 1)} + manager.SetTaskStore(taskStore) + callback := makeDelegateAnnounceCallback(manager, messageBus) + completionID := uuid.New() + + done := make(chan struct{}) + go func() { + callback( + "session-1", + []tools.AnnounceQueueItem{{ + SubagentID: "task-1", + CompletionID: completionID, + DurablyPersisted: true, + Label: "probe", + Status: tools.TaskStatusCompleted, + Result: "done", + }}, + tools.AnnounceMetadata{ + OriginChatID: "chat-1", + OriginSessionKey: "session-1", + OriginTenantID: uuid.New(), + RootAgentID: uuid.New(), + ParentAgent: "root", + }, + ) + close(done) + }() + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("batched subagent announcement blocked on a full inbound bus") + } + select { + case metadata := <-taskStore.metadata: + if metadata["announcement_status"] != "undelivered" { + t.Fatalf("announcement metadata = %#v, want undelivered", metadata) + } + case <-time.After(time.Second): + t.Fatal("batched missed announcement was not recorded after bus saturation") + } +} diff --git a/cmd/gateway_subagent_recovery_test.go b/cmd/gateway_subagent_recovery_test.go new file mode 100644 index 00000000..47669925 --- /dev/null +++ b/cmd/gateway_subagent_recovery_test.go @@ -0,0 +1,65 @@ +package cmd + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +type scriptedSubagentTaskRecovery struct { + mu sync.Mutex + failures int + recovered int64 + calls int +} + +func (s *scriptedSubagentTaskRecovery) RecoverInterrupted(ctx context.Context) (int64, error) { + if !store.IsMasterScope(ctx) || store.TenantIDFromContext(ctx) != store.MasterTenantID { + return 0, errors.New("recovery did not receive explicit master scope") + } + s.mu.Lock() + defer s.mu.Unlock() + s.calls++ + if s.calls <= s.failures { + return 0, errors.New("database temporarily unavailable") + } + return s.recovered, nil +} + +func (s *scriptedSubagentTaskRecovery) callCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.calls +} + +func TestRecoverInterruptedSubagentTasksRetriesBeforeTraffic(t *testing.T) { + recovery := &scriptedSubagentTaskRecovery{failures: 2, recovered: 3} + stores := &store.Stores{SubagentTaskRecovery: recovery} + recovered, err := recoverInterruptedSubagentTasks( + context.Background(), stores, time.Millisecond, + ) + if err != nil { + t.Fatalf("recoverInterruptedSubagentTasks: %v", err) + } + if recovered != 3 || recovery.callCount() != 3 { + t.Fatalf("recovery = (%d rows, %d calls), want (3, 3)", recovered, recovery.callCount()) + } +} + +func TestRecoverInterruptedSubagentTasksStopsOnShutdown(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + recovery := &scriptedSubagentTaskRecovery{failures: 1} + stores := &store.Stores{SubagentTaskRecovery: recovery} + _, err := recoverInterruptedSubagentTasks(ctx, stores, time.Hour) + if !errors.Is(err, context.Canceled) { + t.Fatalf("recovery error = %v, want context.Canceled", err) + } + if recovery.callCount() != 1 { + t.Fatalf("recovery calls = %d, want 1", recovery.callCount()) + } +} diff --git a/docs/00-architecture-overview.md b/docs/00-architecture-overview.md index 6b91091d..bf4c553b 100644 --- a/docs/00-architecture-overview.md +++ b/docs/00-architecture-overview.md @@ -115,7 +115,7 @@ flowchart TD | `internal/skills/` | SKILL.md loader (5-tier hierarchy) + BM25 search + hot-reload via fsnotify | | `internal/channels/` | Channel manager + adapters: Telegram (forum topics, STT, bot commands), Feishu/Lark (streaming cards, media), Zalo OA, Zalo Personal, Discord, WhatsApp, Slack | | `internal/mcp/` | MCP server bridge (stdio, SSE, streamable-HTTP transports) | -| `internal/scheduler/` | Lane-based concurrency control (main, subagent, cron, team lanes) with per-session serialization. Per-edition rate limits (`MaxSubagentConcurrent`, `MaxSubagentDepth`) with tenant-scoped concurrency | +| `internal/scheduler/` | Lane-based concurrency control (main, subagent, cron, team lanes) with per-session serialization. Self-spawn and Agent Link callbacks additionally use process-wide child-run admission with per-root limits | | `internal/memory/` | Memory system (pgvector hybrid search) | | `internal/subagent/` | Subagent lifecycle: spawn, roster, task persistence (subagent_tasks table), announce queue (producer-consumer), auto-retry, per-edition rate limiting | | `internal/permissions/` | RBAC policy engine (admin, operator, viewer roles) | @@ -192,7 +192,7 @@ flowchart TD - **SSRF hardening (HTTPHandler)**: Caller supplies net.Dialer pinning resolved IP, blocking loopback/link-local/private ranges; no HTTP redirects (CheckRedirect returns ErrUseLastResponse) - **Auth header encryption**: `Authorization` + other sensitive fields in cfg.Config["headers"] encrypted at rest via AES-256-GCM; decrypted only at HTTP send-time - **Audit logging**: All hook invocations logged to `hook_executions` table (encrypted, PII-redacted) with dedup_key for idempotency -- **Loop-depth guard (M5)**: SubagentStart checks recursion depth; max 3 levels prevents infinite delegation chains +- **Hook recursion guard (M5)**: Hook dispatch tracks recursive hook re-entry and rejects it after 3 levels. Agent Link delegation uses separate admission, concurrency, and timeout safeguards. - **Circuit breaker**: Auto-disables hook after 3 consecutive failures in recent window (C4 mitigation) ### Pipeline Integration diff --git a/docs/01-agent-loop.md b/docs/01-agent-loop.md index e88e3d78..998bd471 100644 --- a/docs/01-agent-loop.md +++ b/docs/01-agent-loop.md @@ -111,7 +111,7 @@ Agents support three orchestration modes that determine which inter-agent tools - **Use case**: Agent with linked delegate targets - **Tools available**: `spawn`, `delegate` (dispatch to linked agents) - **Tools hidden**: `team_tasks` -- **Resolution**: When `agent_links` table has rows with source = this agent +- **Resolution**: When `agent_links` grants this agent an effective outbound delegation edge ### ModeTeam - **Use case**: Agent in a team (multiple agents collaborating) @@ -121,7 +121,9 @@ Agents support three orchestration modes that determine which inter-agent tools **Mode Resolution Priority**: Team > Delegate > Spawn -The system prompt includes relevant details for each mode (delegate targets, team context, shared workspace paths). +The system prompt includes relevant details for each mode. Agent Team runs use +their existing shared workspace. Agent Link runs receive only delegation-scoped +`inputs/` and `outputs/` aliases. --- diff --git a/docs/03-tools-system.md b/docs/03-tools-system.md index cb68c795..5303e70d 100644 --- a/docs/03-tools-system.md +++ b/docs/03-tools-system.md @@ -144,7 +144,7 @@ Memory layers: L1 (`memory_search`) returns ranked abstracts; L2 (`memory_expand | Tool | Description | |---|---| -| `delegate` | Inter-agent task delegation via agent_links (async/sync modes with timeout) | +| `delegate` | Inter-agent task delegation via `agent_links` (async/sync, optional structured input files, isolated output publication) | ### Teams (`group:team`) @@ -170,6 +170,19 @@ Memory layers: L1 (`memory_search`) returns ranked abstracts; L2 (`memory_expand | `read_document` | Extract and analyze documents (PDF, DOCX, images). Supports local-first extraction via pdftotext/pandoc before falling back to cloud vision (opt-in via config) | | `read_video` | Analyze/transcribe video content | +Image attachments keep absolute paths only in internal `MediaRef` storage. +Model-visible `` tags expose an exact media ID plus a logical path +such as `.uploads/photo.jpg` or delegated `inputs/photo.jpg`. Image tools resolve +those values against the active workspace; they never infer another agent's +absolute workspace path. Unknown image IDs fail instead of falling back to an +unrelated image. Exact `media_id` lookup is an in-process convenience; Claude +CLI/MCP uses the workspace-confined logical `path` under signed bridge context. + +For Agent Link artifact runs, Claude CLI MCP requests carry a signed delegation +ID and staged-input root. Bridge tools therefore apply the same read-only +`inputs/` policy as in-process tools, and generated media remains inside +`outputs/` until the delegation manifest is validated and atomically published. + ### Skills & Content | Tool | Description | @@ -613,13 +626,51 @@ stateDiagram-v2 | Constraint | Default | |---|---| -| Max concurrent | 8 (across all parents) | +| Max concurrent | 20 executing descendants per root agent | | Max spawn depth | 1 | | Max children per agent | 5 | | Archive after | 60 min | | Max iterations | 20 per subagent | -**Actions:** `spawn` (async, returns immediately), `run` (sync, blocks until done), `list`, `cancel` (by ID / `"all"` / `"last"`), `steer` (cancel + respawn with new message). +`maxSpawnDepth` applies only to an agent's own subagent tree. Agent Link +delegation starts a new tree whose spawn depth, concurrency, fanout, retry, and +model settings are resolved from the target agent. + +**Actions:** `spawn(action="spawn", mode="async"|"sync")`, `get`, `list`, +`cancel` (by ID / `"all"` / `"last"`), `steer` (cancel + respawn with new +message), and `wait`. + +An accepted async spawn returns its short runtime `task_id` plus a durable +`completion_id`. Terminal status, the full text result, and workspace-safe +logical media descriptors are written to `subagent_tasks` before the parent +announcement is attempted. Announcement delivery retries a bounded number of +times; if the inbound queue is still full or the gateway restarts after +terminal persistence, the owning root agent can recover the result and logical +file paths with: + +```text +spawn(action="get", completion_id="") +``` + +The lookup is scoped by tenant and immutable root-agent UUID. A same-key agent +created later, another agent in the tenant, and delegation completion rows +cannot satisfy the lookup. + +Terminal persistence uses per-attempt database deadlines and a longer bounded +retry window than announcement delivery. If the database remains unavailable +for the entire window, GoClaw does not falsely mark the announcement as +delivered; a live announcement is still attempted. On the next startup that +can reach the database, every non-terminal completion row left by the previous +process is marked `failed` with an interruption reason, so it cannot remain +`queued` or `running` forever. The single-process gateway retries this +reconciliation before accepting traffic when the database is temporarily +unavailable. Graceful gateway shutdown drains this completion lifecycle before +provider and database teardown. + +Self-spawn and Agent Link callbacks also share a process safety cap (Standard: +32; Lite: 2) and a bounded pending queue of 128. Inside an Agent Link artifact +run, async `spawn` and async nested `delegate` are rejected; configured +synchronous descendants complete before the outer artifact is published. Subagents share the same `SecureCLIStore` as their parent — the credentialed binary gate cannot be bypassed by delegating exec to a child. diff --git a/docs/09-security.md b/docs/09-security.md index 1e7d65d0..809ce0b0 100644 --- a/docs/09-security.md +++ b/docs/09-security.md @@ -461,13 +461,19 @@ Browser pairing allows web UI clients to authenticate without full admin credent ## 12. Delegation Security -Agent delegation is protected through delegation history tracking and concurrency controls. +Agent delegation is protected through isolated artifact exchange, directional +link permission checks, and bounded child-run admission. | Control | Scope | Description | |---------|-------|-------------| -| Per-agent load cap | B (all sources) | `other_config.max_delegation_load` limits total concurrent delegations targeting B | +| Artifact boundary | One Agent Link run | Staged inputs are read-only; the delegatee writes only to ephemeral outputs that are validated before atomic publish | +| Per-root limit | One root agent's self-spawn tree | Agent `subagents.maxConcurrent`, default 20 | +| Process safety cap | All self-spawn and Agent Link callbacks | Standard 32 / Lite 2, with at most 128 independent pending chains | +| Nested lifetime | Agent Link artifact run | Async descendants are rejected; sync descendants complete before publication | +| Durable async result | Source/root agent UUID within one tenant | Terminal text and logical media paths use deadline-bounded persistence retries before announcement; single-process startup retries pre-traffic recovery that marks previous-process non-terminal rows failed, and successful terminal writes remain explicitly retrievable by completion UUID | -When concurrency limits are hit, the error message is written for LLM reasoning: *"Agent at capacity (5/5). Try a different agent or handle it yourself."* +`agent_links.max_concurrent` is retained as compatibility metadata and is not +currently enforced by runtime admission. --- diff --git a/docs/11-agent-teams.md b/docs/11-agent-teams.md index 4b752126..8b456651 100644 --- a/docs/11-agent-teams.md +++ b/docs/11-agent-teams.md @@ -579,10 +579,17 @@ Each delegation link (lead→member) has its own settings: | Layer | Scope | Default | |-------|-------|---------| -| Per-link | Simultaneous delegations from lead to a specific member | 3 | -| Per-agent | Total concurrent delegations targeting any single member | 5 | +| Per-root agent | Executing self-spawn descendants across the full tree | 20 | +| Process safety cap | Executing self-spawn and Agent Link callbacks | Standard 32 / Lite 2 | +| Pending admission | Independent child-run chains waiting for capacity | 128 | -When limits are hit, the error message is written for LLM reasoning: "Agent at capacity (5/5). Try a different agent or handle it yourself." +Each Agent Link delegation begins a new target-owned spawn tree. The source +agent's `maxSpawnDepth` does not limit the target agent; any subagents created by +the target use the target's own subagent settings. + +`agent_links.max_concurrent` remains serialized compatibility metadata and is +not currently an admission limit. Agent Team task scheduling keeps its existing +lane/workspace contract. --- @@ -593,7 +600,7 @@ Agent links define directed delegation relationships between agents, separate fr - **Target agent** (can receive delegations from) - **Direction** (outbound from source, inbound to target) - **Team context** (optional `team_id` if created by team setup) -- **Concurrency limit** (max simultaneous delegations for this link) +- **Reserved concurrency metadata** (`max_concurrent`; retained for compatibility, not currently enforced) **Team-created links** are automatically created when a team is set up (lead → each member). Links remain even if the team is deleted or members are removed, allowing manual cleanup. @@ -601,9 +608,48 @@ Agent links define directed delegation relationships between agents, separate fr ## 12. Delegation Context -### SenderID Clearing +### Authorization and artifact scope -In sync delegations, the delegate agent's context has the `senderID` cleared. This is critical because delegations are system-initiated — the delegate should not inherit the caller's group writer permissions, which would incorrectly deny file writes. Each delegate agent has its own writer list. +Agent Link dispatch preserves the caller's authorization scope for policy +checks but attenuates filesystem authority. The delegatee receives staged +read-only copies through `inputs/` and writes only to `outputs/`. After the run, +regular-file, path, symlink, count, and size validation completes before an +atomic publication to the caller's `.delegations//` directory. +Agent Team shared-workspace behavior is unchanged. + +Successful publications persist with `manifest.json` and logical `outputs/...` +entries. Failed/cancelled exchanges are retained for 60 minutes for diagnostics +and removed by a restart-safe janitor whose per-sweep item and filesystem-entry +work are both bounded. Traces correlate by +`delegation_id` and record only logical paths, sizes, hashes, status, and +timestamps—never physical exchange or caller host paths. + +Async Agent Link dispatch also creates a durable completion row before the +child-run ticket is activated. The full terminal result and logical published +artifact paths are persisted before the automatic parent announcement. If +bounded announcement retries cannot enter the inbound queue, the source agent +can retrieve the result after the run—or after a gateway restart—with: + +```text +delegate(action="get", delegation_id="") +``` + +Retrieval is scoped by tenant and the immutable UUID of the source agent. It +does not grant the target agent, another linked source, or a later same-key +agent access to the result. Delegation completion rows are excluded from +`/subagents` and self-clone roster queries. + +Terminal database writes use per-attempt deadlines plus an extended bounded +retry window. If the database remains unavailable for that whole window, the +live announcement is still attempted but is not recorded as durably delivered; +a crash during the same outage may prevent the terminal result from being +recorded. On the next startup that can reach the database, every non-terminal +completion row from the previous process is marked `failed` with an interruption +reason instead of remaining `queued` or `running` forever. The gateway retries +that reconciliation before accepting traffic if the database is temporarily +unavailable. The current runtime assumes one active gateway process. Graceful +shutdown waits for accepted completion persistence before tearing down runtime +dependencies. ### Trace Linking diff --git a/docs/14-skills-runtime.md b/docs/14-skills-runtime.md index 21e705f7..8e3580d7 100644 --- a/docs/14-skills-runtime.md +++ b/docs/14-skills-runtime.md @@ -166,13 +166,20 @@ Uploaded files (from web chat, Telegram, Discord, etc.) are persisted to: Uploads without a usable original filename fall back to `{uuid}.{ext}`. Legacy media refs may still resolve from `.media/{sessionHash}/{uuid}.{ext}`. -The `enrichDocumentPaths()` function injects the full path into `` tags: +The `enrichDocumentPaths()` function injects the exact media ID and a logical +path relative to the active agent workspace into `` tags: ``` - + ``` -Agents can read these files directly via exec — no copy to `/tmp` needed. For archive uploads such as `.zip`, inspect or extract with commands like `unzip -l ""` or `unzip -q "" -d `. +Normal agent runs can read these workspace files directly via `exec` — no copy +to `/tmp` is needed. Agent Link delegations instead receive selected files as +read-only `inputs/...` paths in an isolated delegation exchange. Their `exec` +calls fail closed unless an active sandbox is available, and generated files +must be written under `outputs/` for validation and publication back to the +caller. For archive uploads such as `.zip`, inspect or extract them within the +authorized workspace or delegation paths. --- diff --git a/internal/agent/delegation_artifact_workspace.go b/internal/agent/delegation_artifact_workspace.go new file mode 100644 index 00000000..499c1f18 --- /dev/null +++ b/internal/agent/delegation_artifact_workspace.go @@ -0,0 +1,141 @@ +package agent + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + + "github.com/google/uuid" + + "github.com/nextlevelbuilder/goclaw/internal/providers" + "github.com/nextlevelbuilder/goclaw/internal/tracing" +) + +func isExistingRealDirectory(path string) bool { + if path == "" { + return false + } + info, err := os.Lstat(path) + return err == nil && info.IsDir() && info.Mode()&os.ModeSymlink == 0 +} + +func validateDelegationArtifactWorkspace(delegationID, inputsPath, outputsPath string) bool { + parsedID, err := uuid.Parse(delegationID) + if err != nil || parsedID == uuid.Nil || + !isExistingRealDirectory(inputsPath) || + !isExistingRealDirectory(outputsPath) { + return false + } + inputsReal, err := filepath.EvalSymlinks(filepath.Clean(inputsPath)) + if err != nil { + return false + } + outputsReal, err := filepath.EvalSymlinks(filepath.Clean(outputsPath)) + if err != nil { + return false + } + exchangeRoot := filepath.Dir(inputsReal) + return filepath.Base(inputsReal) == "inputs" && + filepath.Base(outputsReal) == "outputs" && + filepath.Dir(outputsReal) == exchangeRoot && + filepath.Base(exchangeRoot) == parsedID.String() && + filepath.Base(filepath.Dir(exchangeRoot)) == "delegations" && + filepath.Base(filepath.Dir(filepath.Dir(exchangeRoot))) == "collaboration" +} + +func delegationArtifactTextRedactor(req *RunRequest) tracing.TextRedactor { + if req == nil || req.RunKind != "delegate" || + req.DelegateInputsPath == "" || req.DelegateOutputsPath == "" { + return nil + } + type replacement struct { + value string + alias string + } + var replacements []replacement + seen := make(map[string]struct{}) + add := func(value, alias string) { + for _, variant := range []string{ + value, + filepath.ToSlash(value), + strings.Trim(strconv.Quote(value), `"`), + } { + if variant == "" { + continue + } + if _, exists := seen[variant]; exists { + continue + } + seen[variant] = struct{}{} + replacements = append(replacements, replacement{value: variant, alias: alias}) + } + } + add(req.DelegateInputsPath, "inputs") + add(req.DelegateOutputsPath, "outputs") + if root := filepath.Dir(req.DelegateInputsPath); root == filepath.Dir(req.DelegateOutputsPath) { + add(root, "delegation exchange") + } + sort.Slice(replacements, func(i, j int) bool { + return len(replacements[i].value) > len(replacements[j].value) + }) + pairs := make([]string, 0, len(replacements)*2) + for _, replacement := range replacements { + pairs = append(pairs, replacement.value, replacement.alias) + } + replacer := strings.NewReplacer(pairs...) + return replacer.Replace +} + +func withDelegationArtifactTextRedactor(ctx context.Context, req *RunRequest) context.Context { + return tracing.WithTextRedactor(ctx, delegationArtifactTextRedactor(req)) +} + +func redactDelegationAgentEvent(req *RunRequest, event AgentEvent) AgentEvent { + redactor := delegationArtifactTextRedactor(req) + event.Payload = tracing.RedactValueWith(redactor, event.Payload) + return event +} + +func redactDelegationMessage(req *RunRequest, message providers.Message) providers.Message { + redactor := delegationArtifactTextRedactor(req) + if redactor == nil { + return message + } + message.Content = redactor(message.Content) + message.Thinking = redactor(message.Thinking) + if len(message.RawAssistantContent) > 0 { + message.RawAssistantContent = json.RawMessage(redactor(string(message.RawAssistantContent))) + } + for i := range message.ToolCalls { + if arguments, ok := tracing.RedactValueWith(redactor, message.ToolCalls[i].Arguments).(map[string]any); ok { + message.ToolCalls[i].Arguments = arguments + } + message.ToolCalls[i].ParseError = redactor(message.ToolCalls[i].ParseError) + } + // Pre-publication paths are ephemeral and cannot be durable session media. + message.MediaRefs = nil + return message +} + +func redactDelegationRunResult(req *RunRequest, result *RunResult) *RunResult { + if result == nil { + return nil + } + redactor := delegationArtifactTextRedactor(req) + if redactor == nil { + return result + } + result.Content = redactor(result.Content) + result.Thinking = redactor(result.Thinking) + result.LastBlockReply = redactor(result.LastBlockReply) + for i := range result.Deliverables { + result.Deliverables[i] = redactor(result.Deliverables[i]) + } + // The outer artifact publisher is the only egress path. + result.Media = nil + return result +} diff --git a/internal/agent/loop_context.go b/internal/agent/loop_context.go index b1be7c47..53df6375 100644 --- a/internal/agent/loop_context.go +++ b/internal/agent/loop_context.go @@ -27,6 +27,32 @@ type contextSetupResult struct { // values needed by the agent loop and tool execution. Also runs input guard and message // truncation. Returns error only if input guard blocks the message. func (l *Loop) injectContext(ctx context.Context, req *RunRequest) (contextSetupResult, error) { + isArtifactDelegation := req.RunKind == "delegate" + + // A nested run must not inherit filesystem, Team, media, or delegation + // authority from its caller. Install explicit empty values before resolving + // this run's own scope. + ctx = store.WithRunContext(ctx, nil) + ctx = tools.WithToolWorkspace(ctx, "") + ctx = tools.WithToolTeamWorkspace(ctx, "") + ctx = tools.WithToolTeamRoot(ctx, "") + ctx = tools.WithToolTeamID(ctx, "") + ctx = tools.WithTeamTaskID(ctx, "") + ctx = tools.WithLeaderAgentID(ctx, "") + ctx = tools.WithTenantAllowedPaths(ctx, nil) + ctx = tools.WithWorkspaceChannel(ctx, "") + ctx = tools.WithWorkspaceChatID(ctx, "") + ctx = tools.WithDelegationID(ctx, "") + ctx = tools.WithDelegationArtifactInputs(ctx, "") + ctx = tools.WithRunKind(ctx, "") + ctx = tools.WithRunMediaPaths(ctx, nil) + ctx = tools.WithRunMediaNames(ctx, nil) + ctx = tools.WithMediaImages(ctx, nil) + ctx = tools.WithMediaImageRefs(ctx, nil) + ctx = tools.WithMediaDocRefs(ctx, nil) + ctx = tools.WithMediaAudioRefs(ctx, nil) + ctx = tools.WithMediaVideoRefs(ctx, nil) + // Inject agent UUID + key into context for tool routing if l.agentUUID != uuid.Nil { ctx = store.WithAgentID(ctx, l.agentUUID) @@ -87,7 +113,7 @@ func (l *Loop) injectContext(ctx context.Context, req *RunRequest) (contextSetup ctx = tools.WithTenantToolSettings(ctx, l.tenantToolSettings) } // Inject tenant-specific allowed paths for filesystem tools. - if len(l.tenantAllowedPaths) > 0 { + if !isArtifactDelegation && len(l.tenantAllowedPaths) > 0 { ctx = tools.WithTenantAllowedPaths(ctx, l.tenantAllowedPaths) } // Inject channel type into context for tools (e.g. message tool needs it for Zalo group routing) @@ -150,6 +176,9 @@ func (l *Loop) injectContext(ctx context.Context, req *RunRequest) (contextSetup if req.DelegationID != "" { ctx = tools.WithDelegationID(ctx, req.DelegationID) } + if req.RunKind != "" { + ctx = tools.WithRunKind(ctx, req.RunKind) + } // --- Per-user setup: file seeding + workspace resolution --- // Uses userSetups sync.Map to track both concerns atomically per user. @@ -163,7 +192,7 @@ func (l *Loop) injectContext(ctx context.Context, req *RunRequest) (contextSetup // Layer order: tenant → team → project (future) → user/chat // Two entry modes: solo agent (base = l.workspace) or team context (base = l.dataDir). // Result is always a single folder set via WithToolWorkspace. - if l.workspace != "" && req.UserID != "" { + if !isArtifactDelegation && l.workspace != "" && req.UserID != "" { ws := setup.workspace if ws == "" { ws = l.workspace @@ -197,12 +226,21 @@ func (l *Loop) injectContext(ctx context.Context, req *RunRequest) (contextSetup effectiveWorkspace = l.workspace } ctx = tools.WithToolWorkspace(ctx, effectiveWorkspace) - } else if l.workspace != "" { + } else if !isArtifactDelegation && l.workspace != "" { ctx = tools.WithToolWorkspace(ctx, l.workspace) } + if isArtifactDelegation { + if req.TeamWorkspace != "" || + !validateDelegationArtifactWorkspace(req.DelegationID, req.DelegateInputsPath, req.DelegateOutputsPath) { + return contextSetupResult{}, fmt.Errorf("invalid delegation artifact workspace") + } + ctx = tools.WithDelegationArtifactInputs(ctx, req.DelegateInputsPath) + ctx = tools.WithToolWorkspace(ctx, req.DelegateOutputsPath) + } + // Team workspace: dispatched task overrides default workspace. - if req.TeamWorkspace != "" { + if !isArtifactDelegation && req.TeamWorkspace != "" { if err := os.MkdirAll(req.TeamWorkspace, 0755); err != nil { // See note above on loop_context user workspace fallback. A broken // req.TeamWorkspace would otherwise become cmd.Dir and surface as @@ -214,7 +252,7 @@ func (l *Loop) injectContext(ctx context.Context, req *RunRequest) (contextSetup ctx = tools.WithToolWorkspace(ctx, req.TeamWorkspace) } } - if req.TeamID != "" { + if !isArtifactDelegation && req.TeamID != "" { ctx = tools.WithToolTeamID(ctx, req.TeamID) // Team root for dispatched tasks: resolve the UserChatLayer-stripped root // so the dispatched agent can still read peer-scoped files in the same team. @@ -228,7 +266,7 @@ func (l *Loop) injectContext(ctx context.Context, req *RunRequest) (contextSetup ctx = tools.WithToolTeamRoot(ctx, teamRoot) } } - if req.LeaderAgentID != "" { + if !isArtifactDelegation && req.LeaderAgentID != "" { ctx = tools.WithLeaderAgentID(ctx, req.LeaderAgentID) } @@ -237,14 +275,14 @@ func (l *Loop) injectContext(ctx context.Context, req *RunRequest) (contextSetup var resolvedTeamSettings json.RawMessage // Dispatched tasks already have TeamWorkspace set but still need team settings // for TeamIsolated flag. Fetch by explicit TeamID in that branch. - if req.TeamWorkspace != "" && req.TeamID != "" && l.teamStore != nil { + if !isArtifactDelegation && req.TeamWorkspace != "" && req.TeamID != "" && l.teamStore != nil { if teamUUID, err := uuid.Parse(req.TeamID); err == nil { if team, _ := l.teamStore.GetTeam(ctx, teamUUID); team != nil { resolvedTeamSettings = team.Settings } } } - if req.TeamWorkspace == "" && l.teamStore != nil && l.agentUUID != uuid.Nil { + if !isArtifactDelegation && req.TeamWorkspace == "" && l.teamStore != nil && l.agentUUID != uuid.Nil { if team, _ := l.teamStore.GetTeamForAgent(ctx, l.agentUUID); team != nil { resolvedTeamSettings = team.Settings wsChat := req.ChatID @@ -285,7 +323,16 @@ func (l *Loop) injectContext(ctx context.Context, req *RunRequest) (contextSetup } // V3 workspace: resolve once, set immutable context. - { + if isArtifactDelegation { + ctx = workspace.WithContext(ctx, &workspace.WorkspaceContext{ + ActivePath: req.DelegateOutputsPath, + Scope: workspace.ScopeDelegate, + MemoryScope: "user", + KGScope: "user", + OwnerID: req.UserID, + EnforcementLabel: workspace.DefaultEnforcementLabel(workspace.ScopeDelegate, false), + }) + } else { var teamIDPtr *string if req.TeamID != "" { teamIDPtr = &req.TeamID @@ -386,6 +433,10 @@ func (l *Loop) injectContext(ctx context.Context, req *RunRequest) (contextSetup } // Extract resolved credential user ID (set earlier via WithCredentialUserID, empty if not resolved). credUserID := store.ExplicitCredentialUserIDFromContext(ctx) + tenantAllowedPaths := l.tenantAllowedPaths + if isArtifactDelegation { + tenantAllowedPaths = nil + } rc := &store.RunContext{ AgentID: l.agentUUID, AgentKey: l.id, @@ -419,9 +470,10 @@ func (l *Loop) injectContext(ctx context.Context, req *RunRequest) (contextSetup WorkspaceChatID: effectiveWorkspaceChatID, TeamIsolated: resolvedTeamSettings != nil && !tools.IsSharedWorkspace(resolvedTeamSettings), TeamTaskID: req.TeamTaskID, + DelegationID: req.DelegationID, LeaderAgentID: tools.LeaderAgentIDFromCtx(ctx), AgentToolKey: l.id, - TenantAllowedPaths: l.tenantAllowedPaths, + TenantAllowedPaths: tenantAllowedPaths, } ctx = store.WithRunContext(ctx, rc) diff --git a/internal/agent/loop_context_collaboration_test.go b/internal/agent/loop_context_collaboration_test.go new file mode 100644 index 00000000..e394df0e --- /dev/null +++ b/internal/agent/loop_context_collaboration_test.go @@ -0,0 +1,235 @@ +package agent + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/uuid" + + "github.com/nextlevelbuilder/goclaw/internal/providers" + "github.com/nextlevelbuilder/goclaw/internal/store" + "github.com/nextlevelbuilder/goclaw/internal/tools" + "github.com/nextlevelbuilder/goclaw/internal/workspace" +) + +func newArtifactRunRequest(t *testing.T, root string) *RunRequest { + t.Helper() + delegationID := uuid.NewString() + exchange := filepath.Join(root, "collaboration", "delegations", delegationID) + inputs := filepath.Join(exchange, "inputs") + outputs := filepath.Join(exchange, "outputs") + if err := os.MkdirAll(inputs, 0750); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(outputs, 0750); err != nil { + t.Fatal(err) + } + return &RunRequest{ + SessionKey: "agent:worker:delegate:test", + RunKind: "delegate", + DelegationID: delegationID, + DelegateInputsPath: inputs, + DelegateOutputsPath: outputs, + } +} + +func newArtifactTestLoop(root string) *Loop { + return NewLoop(LoopConfig{ + ID: "worker", + AgentUUID: uuid.New(), + TenantID: store.MasterTenantID, + Workspace: filepath.Join(root, "agents", "worker"), + DataDir: root, + Sessions: &nopSessionStore{}, + }) +} + +func TestInjectContext_UsesOnlyDelegationArtifactWorkspace(t *testing.T) { + root := t.TempDir() + req := newArtifactRunRequest(t, root) + parentCtx := tools.WithToolTeamWorkspace(context.Background(), filepath.Join(root, "teams", "stale")) + parentCtx = tools.WithToolTeamID(parentCtx, uuid.NewString()) + parentCtx = tools.WithTenantAllowedPaths(parentCtx, []string{filepath.Join(root, "tenant-allowed")}) + parentCtx = tools.WithDelegationArtifactInputs(parentCtx, filepath.Join(root, "stale-inputs")) + parentCtx = tools.WithRunMediaPaths(parentCtx, []string{"/stale/upload.txt"}) + + setup, err := newArtifactTestLoop(root).injectContext(parentCtx, req) + if err != nil { + t.Fatalf("injectContext: %v", err) + } + if got := tools.ToolWorkspaceFromCtx(setup.ctx); got != req.DelegateOutputsPath { + t.Fatalf("workspace = %q, want outputs %q", got, req.DelegateOutputsPath) + } + if got := tools.DelegationArtifactInputsFromCtx(setup.ctx); got != req.DelegateInputsPath { + t.Fatalf("inputs = %q, want %q", got, req.DelegateInputsPath) + } + if got := tools.ToolTeamWorkspaceFromCtx(setup.ctx); got != "" { + t.Fatalf("parent Team workspace survived: %q", got) + } + if got := tools.ToolTeamIDFromCtx(setup.ctx); got != "" { + t.Fatalf("parent Team ID survived: %q", got) + } + if got := tools.TenantAllowedPathsFromCtx(setup.ctx); len(got) != 0 { + t.Fatalf("parent tenant allowed paths survived: %v", got) + } + if got := tools.RunMediaPathsFromCtx(setup.ctx); len(got) != 0 { + t.Fatalf("parent media paths survived: %v", got) + } + wc := workspace.FromContext(setup.ctx) + if wc == nil || wc.Scope != workspace.ScopeDelegate || wc.ActivePath != req.DelegateOutputsPath { + t.Fatalf("workspace context = %#v", wc) + } +} + +func TestInjectContext_DelegationSetupFailsClosed(t *testing.T) { + root := t.TempDir() + req := newArtifactRunRequest(t, root) + req.DelegateOutputsPath = filepath.Join(root, "missing") + if _, err := newArtifactTestLoop(root).injectContext(context.Background(), req); err == nil { + t.Fatal("missing artifact workspace fell back to personal workspace") + } + + outside := filepath.Join(root, "outside") + if err := os.MkdirAll(outside, 0750); err != nil { + t.Fatal(err) + } + link := filepath.Join(root, "output-link") + if err := os.Symlink(outside, link); err != nil { + t.Fatal(err) + } + req.DelegateOutputsPath = link + if _, err := newArtifactTestLoop(root).injectContext(context.Background(), req); err == nil { + t.Fatal("symlink artifact workspace fell back to personal workspace") + } + + req = newArtifactRunRequest(t, root) + req.DelegationID = uuid.NewString() + if _, err := newArtifactTestLoop(root).injectContext(context.Background(), req); err == nil { + t.Fatal("exchange path not bound to delegation ID was accepted") + } +} + +func TestInjectContext_DelegatePreservesAuthorizationScope(t *testing.T) { + root := t.TempDir() + req := newArtifactRunRequest(t, root) + req.UserID = "group:telegram:-100123" + req.SenderID = "386246614" + req.Role = "viewer" + req.Channel = "delegate" + req.ChannelType = "telegram" + req.ChatID = "-100123" + req.PeerKind = "group" + req.WorkspaceChannel = "telegram-main" + req.WorkspaceChatID = "-100123" + + setup, err := newArtifactTestLoop(root).injectContext(context.Background(), req) + if err != nil { + t.Fatalf("injectContext: %v", err) + } + if got := store.UserIDFromContext(setup.ctx); got != req.UserID { + t.Fatalf("UserID = %q, want %q", got, req.UserID) + } + if got := store.SenderIDFromContext(setup.ctx); got != req.SenderID { + t.Fatalf("SenderID = %q, want %q", got, req.SenderID) + } + if got := store.RoleFromContext(setup.ctx); got != req.Role { + t.Fatalf("Role = %q, want %q", got, req.Role) + } +} + +func TestInjectContext_TeamDispatchKeepsSharedWorkspaceContract(t *testing.T) { + root := t.TempDir() + teamWorkspace := filepath.Join(root, "teams", "team-a", "shared") + if err := os.MkdirAll(teamWorkspace, 0750); err != nil { + t.Fatal(err) + } + req := &RunRequest{ + SessionKey: "agent:worker:team:test", + TeamWorkspace: teamWorkspace, + TeamID: uuid.NewString(), + TeamTaskID: uuid.NewString(), + LeaderAgentID: uuid.NewString(), + } + + setup, err := newArtifactTestLoop(root).injectContext(context.Background(), req) + if err != nil { + t.Fatalf("injectContext: %v", err) + } + if got := tools.ToolWorkspaceFromCtx(setup.ctx); got != teamWorkspace { + t.Fatalf("active workspace = %q, want Team workspace %q", got, teamWorkspace) + } + if got := tools.ToolTeamWorkspaceFromCtx(setup.ctx); got != teamWorkspace { + t.Fatalf("Team workspace = %q, want %q", got, teamWorkspace) + } + if got := tools.ToolTeamIDFromCtx(setup.ctx); got != req.TeamID { + t.Fatalf("Team ID = %q, want %q", got, req.TeamID) + } + if got := tools.TeamTaskIDFromCtx(setup.ctx); got != req.TeamTaskID { + t.Fatalf("Team task ID = %q, want %q", got, req.TeamTaskID) + } + if got := tools.DelegationArtifactInputsFromCtx(setup.ctx); got != "" { + t.Fatalf("Team dispatch gained artifact inputs: %q", got) + } +} + +func TestDelegationArtifactRedactionCoversEventsSessionsAndResults(t *testing.T) { + req := newArtifactRunRequest(t, t.TempDir()) + hostResult := filepath.Join(req.DelegateOutputsPath, "report.txt") + hostInput := filepath.Join(req.DelegateInputsPath, "source.txt") + + event := redactDelegationAgentEvent(req, AgentEvent{ + Payload: map[string]any{ + "content": "created " + hostResult, + "media": []any{hostResult}, + }, + }) + message := redactDelegationMessage(req, providers.Message{ + Content: "read " + hostInput, + Thinking: "write " + hostResult, + MediaRefs: []providers.MediaRef{{Path: hostResult}}, + ToolCalls: []providers.ToolCall{{ + Arguments: map[string]any{"path": hostResult}, + }}, + RawAssistantContent: json.RawMessage(`{"path":` + mustJSONQuote(t, hostResult) + `}`), + }) + result := redactDelegationRunResult(req, &RunResult{ + Content: "done at " + hostResult, + Thinking: hostInput, + Deliverables: []string{hostResult}, + Media: []MediaResult{{Path: hostResult}}, + }) + + encoded, err := json.Marshal([]any{event, message, result}) + if err != nil { + t.Fatal(err) + } + for _, hostRoot := range []string{ + filepath.Dir(req.DelegateInputsPath), + req.DelegateInputsPath, + req.DelegateOutputsPath, + } { + if strings.Contains(string(encoded), hostRoot) { + t.Fatalf("redacted boundary leaked %q: %s", hostRoot, encoded) + } + } + if len(message.MediaRefs) != 0 || len(result.Media) != 0 { + t.Fatalf("ephemeral media survived: message=%#v result=%#v", message.MediaRefs, result.Media) + } + if !strings.Contains(string(encoded), "inputs/source.txt") || + !strings.Contains(string(encoded), "outputs/report.txt") { + t.Fatalf("logical aliases missing: %s", encoded) + } +} + +func mustJSONQuote(t *testing.T, value string) string { + t.Helper() + encoded, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + return string(encoded) +} diff --git a/internal/agent/loop_finalize.go b/internal/agent/loop_finalize.go index 2e42eff1..07dbf4a7 100644 --- a/internal/agent/loop_finalize.go +++ b/internal/agent/loop_finalize.go @@ -14,7 +14,6 @@ import ( "github.com/nextlevelbuilder/goclaw/internal/i18n" "github.com/nextlevelbuilder/goclaw/internal/providers" "github.com/nextlevelbuilder/goclaw/internal/store" - "github.com/nextlevelbuilder/goclaw/internal/tools" ) // isUserFilePopulated checks if USER.md has been filled with actual user data @@ -48,7 +47,7 @@ func (l *Loop) finalizeRun( // BEFORE sanitize strips them. Covers cases where a tool returned its // artifact via the ForLLM MEDIA: prefix but the agent relayed it as plain // text (e.g. PDF from exec/weasyprint, TTS mp3 paths the LLM quotes back). - if extracted := extractMediaFromContent(rs.finalContent, tools.ToolWorkspaceFromCtx(ctx)); len(extracted) > 0 { + if extracted := extractMediaFromContent(rs.finalContent, l.mediaEgressRoots(ctx)); len(extracted) > 0 { rs.mediaResults = append(rs.mediaResults, extracted...) } @@ -67,8 +66,12 @@ func (l *Loop) finalizeRun( rs.finalContent += "\n\n---\n_" + i18n.T(locale, i18n.MsgSkillNudgePostscript) + "_" } - // 7. Fallback for empty content - if rs.finalContent == "" { + // 7. Fallback only when there is no other deliverable output. Media-only + // runs must remain media-only instead of gaining a visible "..." caption. + hasDeliverableOutput := len(rs.mediaResults) > 0 || + len(req.ForwardMedia) > 0 || + req.ContentSuffix != "" + if rs.finalContent == "" && !hasDeliverableOutput { if len(rs.asyncToolCalls) > 0 { rs.finalContent = "..." } else { @@ -200,6 +203,9 @@ func (l *Loop) finalizeRun( slog.Info("agent loop: NO_REPLY detected, suppressing delivery", "agent", l.id, "session", req.SessionKey) rs.finalContent = "" + if req.ContentSuffix != "" { + rs.finalContent = deduplicateMediaSuffix("", req.ContentSuffix) + } } // 9. Maybe summarize diff --git a/internal/agent/loop_history.go b/internal/agent/loop_history.go index 8d432447..2c4d57b1 100644 --- a/internal/agent/loop_history.go +++ b/internal/agent/loop_history.go @@ -48,6 +48,14 @@ func (l *Loop) buildMessages(ctx context.Context, history []providers.Message, s tools.UserChatLayer(tools.SanitizePathSegment(userID), shared), ) } + if tools.IsDelegationArtifactRun(ctx) { + promptWorkspace = "." + const artifactGuidance = "Delegation workspace: write outputs using ordinary relative paths in the current workspace. Read staged inputs only through inputs/... . Files are returned to the caller only after runtime validation and publication." + if extraSystemPrompt != "" { + extraSystemPrompt += "\n\n" + } + extraSystemPrompt += artifactGuidance + } // Resolve context files once — also detect BOOTSTRAP.md presence. // lightContext: skip loading context files, only inject ExtraSystemPrompt (heartbeat checklist). diff --git a/internal/agent/loop_input_media.go b/internal/agent/loop_input_media.go index 49cb3394..13d51d80 100644 --- a/internal/agent/loop_input_media.go +++ b/internal/agent/loop_input_media.go @@ -65,6 +65,9 @@ func (l *Loop) enrichInputMedia(ctx context.Context, req *RunRequest, messages [ var mediaRefs []providers.MediaRef if len(req.Media) > 0 { mediaRefs = l.persistMedia(req.SessionKey, req.Media, tools.ToolWorkspaceFromCtx(ctx)) + if req.RunKind == "delegate" { + rehomeDelegatedMediaMessage(messages, req.Media, mediaRefs, tools.ToolWorkspaceFromCtx(ctx)) + } // Register persisted text uploads in vault (async, non-blocking). if l.onTextUploaded != nil { @@ -109,46 +112,55 @@ func (l *Loop) enrichInputMedia(ctx context.Context, req *RunRequest, messages [ ctx = l.loadHistoricalImagesForTool(ctx, mediaRefs, messages) } - // 2b. Collect document MediaRefs (historical + current) for read_document tool. + // 2b. Collect image MediaRefs (historical + current) for exact media_id + // resolution by read_image and create_image. + if imageRefs := collectRefsByKind(messages, mediaRefs, "image"); len(imageRefs) > 0 { + ctx = tools.WithMediaImageRefs(ctx, imageRefs) + } + + // 2c. Collect document MediaRefs (historical + current) for read_document tool. if docRefs := collectRefsByKind(messages, mediaRefs, "document"); len(docRefs) > 0 { ctx = tools.WithMediaDocRefs(ctx, docRefs) - // Enrich the last user message with persisted file paths so skills can access - // documents via exec (e.g. pypdf). Only for current-turn refs (just persisted). - l.enrichDocumentPaths(messages, mediaRefs) + // Enrich the last user message with exact IDs and logical workspace paths. + // Only current-turn refs are paired with current-turn tags. + l.enrichDocumentPaths(messages, mediaRefs, tools.ToolWorkspaceFromCtx(ctx)) } - // 2c. Collect audio MediaRefs (historical + current) for read_audio tool. + // 2d. Collect audio MediaRefs (historical + current) for read_audio tool. if audioRefs := collectRefsByKind(messages, mediaRefs, "audio"); len(audioRefs) > 0 { ctx = tools.WithMediaAudioRefs(ctx, audioRefs) - l.enrichAudioIDs(messages, mediaRefs) + l.enrichAudioIDs(messages, mediaRefs, tools.ToolWorkspaceFromCtx(ctx)) } - // 2d. Collect video MediaRefs (historical + current) for read_video tool. + // 2e. Collect video MediaRefs (historical + current) for read_video tool. if videoRefs := collectRefsByKind(messages, mediaRefs, "video"); len(videoRefs) > 0 { ctx = tools.WithMediaVideoRefs(ctx, videoRefs) l.enrichVideoIDs(messages, mediaRefs) } - // 2e. Enrich tags with persisted media IDs so the LLM + // 2f. Enrich tags with persisted media IDs so the LLM // knows images were received and stored (consistent with audio/video enrichment). - l.enrichImageIDs(messages, mediaRefs) + l.enrichImageIDs(messages, mediaRefs, tools.ToolWorkspaceFromCtx(ctx)) - // 2e-ii. In file-ref mode, enrich ALL user messages' image tags with file paths. - // This enables read_image(path=...) for both current and historical images. - if deferToReadImageTool { - l.enrichImagePaths(messages) - } + // 2f-ii. Enrich ALL user messages' image tags with logical file paths. + // This upgrades legacy absolute paths and keeps current and historical media + // references safe regardless of whether images are also attached inline. + l.enrichImagePaths(messages, tools.ToolWorkspaceFromCtx(ctx)) - // 2f. Collect all media file paths for team workspace auto-collect. + // 2g. Collect all media file paths for team workspace auto-collect. // When the leader calls team_tasks(create), these paths are copied to the // team workspace so members can access attached files. - if len(mediaRefs) > 0 && l.mediaStore != nil { + if len(mediaRefs) > 0 { var mediaPaths []string for _, ref := range mediaRefs { // Prefer workspace-local path (.uploads/) over canonical .media/ path. if ref.Path != "" { mediaPaths = append(mediaPaths, ref.Path) - } else if p, err := l.mediaStore.LoadPath(ref.ID); err == nil { + } else if l.mediaStore != nil { + p, err := l.mediaStore.LoadPath(ref.ID) + if err != nil { + continue + } mediaPaths = append(mediaPaths, p) } } @@ -166,3 +178,70 @@ func (l *Loop) enrichInputMedia(ctx context.Context, req *RunRequest, messages [ return ctx, messages, mediaRefs } + +// rehomeDelegatedMediaMessage replaces caller-owned attachment paths with the +// delegatee-owned persisted copies and ensures every imported attachment has a +// media tag. The normal enrichment pass then adds the new IDs and local paths. +func rehomeDelegatedMediaMessage( + messages []providers.Message, + input []bus.MediaFile, + refs []providers.MediaRef, + workspace string, +) { + if len(messages) == 0 || len(refs) == 0 { + return + } + lastUser := -1 + for i := len(messages) - 1; i >= 0; i-- { + if messages[i].Role == "user" { + lastUser = i + break + } + } + if lastUser < 0 { + return + } + + content := messages[lastUser].Content + for i := 0; i < len(input) && i < len(refs); i++ { + if input[i].Path != "" && refs[i].Path != "" { + replacement := logicalWorkspaceMediaPath(workspace, refs[i].Path) + if replacement == "" { + replacement = filepath.Base(refs[i].Path) + } + content = strings.ReplaceAll(content, input[i].Path, replacement) + } + } + + tagCounts := map[string]int{ + "image": strings.Count(content, " 0 { + tagCounts[ref.Kind]-- + continue + } + if content != "" && !strings.HasSuffix(content, "\n") { + content += "\n" + } + content += tag + } + messages[lastUser].Content = content +} diff --git a/internal/agent/loop_media.go b/internal/agent/loop_media.go index 0c7ce4f7..98c10c97 100644 --- a/internal/agent/loop_media.go +++ b/internal/agent/loop_media.go @@ -1,9 +1,12 @@ package agent import ( + "context" "os" "path/filepath" "strings" + + "github.com/nextlevelbuilder/goclaw/internal/tools" ) // parseMediaResult extracts a MediaResult from a tool result string containing "MEDIA:" prefix. @@ -94,8 +97,7 @@ func confineToWorkspace(mediaPath, workspace string) (string, bool) { path = filepath.Join(wsRoot, path) } cleaned := filepath.Clean(path) - info, err := os.Lstat(cleaned) - if err != nil || !info.Mode().IsRegular() { + if err := tools.ValidateRegularFileForRead(cleaned); err != nil { return "", false } resolved, err := filepath.EvalSymlinks(cleaned) @@ -109,18 +111,31 @@ func confineToWorkspace(mediaPath, workspace string) (string, bool) { return cleaned, true } -// confineToAnyRoot accepts mediaPath if it is contained by ANY of the allowed -// roots (each checked with the hardened confineToWorkspace). Used for the -// result.Media egress: a tool's media legitimately lives in the agent -// workspace, the team workspace, OR a tenant-allowed path — the same scopes the -// producing tools (create_*, send_file, delegate) validate against. A path -// outside every root (e.g. /etc/passwd from a prompt-injected path) is rejected, -// so the egress guard holds without dropping legitimate cross-workspace media. +// confineToAnyRoot accepts an absolute mediaPath if it is contained by any +// allowed root. Relative paths are intentionally anchored to the first root, +// which is always the active tool workspace; extra roots authorize explicit +// paths but never change the meaning of an ambiguous relative path. func confineToAnyRoot(mediaPath string, roots []string) (string, bool) { - for _, root := range roots { + if len(roots) == 0 { + return "", false + } + if !filepath.IsAbs(mediaPath) { + return confineToWorkspace(mediaPath, roots[0]) + } + for i, root := range roots { if root == "" { continue } + // The active workspace (index 0) may itself be reached through an + // operator-managed symlink. Extra read roots are authorization + // boundaries and must be real directories, not partner-controlled + // symlink aliases to arbitrary locations. + if i > 0 { + if info, err := os.Lstat(root); err == nil && + (info.Mode()&os.ModeSymlink != 0 || !info.IsDir()) { + continue + } + } if cleaned, ok := confineToWorkspace(mediaPath, root); ok { return cleaned, true } @@ -128,17 +143,43 @@ func confineToAnyRoot(mediaPath string, roots []string) (string, bool) { return "", false } +// mediaEgressRoots returns every read-authorized root that may supply outbound +// media. The active workspace stays first to preserve relative path semantics. +func (l *Loop) mediaEgressRoots(ctx context.Context) []string { + tenantAllowedPaths := tools.TenantAllowedPathsFromCtx(ctx) + candidates := make([]string, 0, 3+len(tenantAllowedPaths)) + candidates = append(candidates, + tools.ToolWorkspaceFromCtx(ctx), + tools.ToolTeamWorkspaceFromCtx(ctx), + tools.ToolTeamRootFromCtx(ctx), + ) + candidates = append(candidates, tenantAllowedPaths...) + + seen := make(map[string]struct{}, len(candidates)) + roots := make([]string, 0, len(candidates)) + for _, root := range candidates { + if root == "" { + continue + } + cleaned := filepath.Clean(root) + if _, ok := seen[cleaned]; ok { + continue + } + seen[cleaned] = struct{}{} + roots = append(roots, cleaned) + } + return roots +} + // extractMediaFromContent scans text for MEDIA: tokens the LLM may echo // in its final response (e.g. when a tool returned the MEDIA: prefix as plain -// text instead of setting Result.Media). Relative paths are resolved against -// workspace. Called before sanitize strips the tokens so the attachments are -// still delivered. +// text instead of setting Result.Media). The first root is the active workspace +// used for relative paths; later roots authorize explicit absolute paths. +// Called before sanitize strips the tokens so the attachments are delivered. // -// Security: only paths accepted by confineToWorkspace are emitted. An LLM cannot -// inject attachments pointing at /etc/passwd, a sibling tenant's workspace, or a -// hallucinated path — the extractor silently drops them. -func extractMediaFromContent(content, workspace string) []MediaResult { - if !strings.Contains(content, "MEDIA:") || workspace == "" { +// Security: only paths accepted by confineToAnyRoot are emitted. +func extractMediaFromContent(content string, roots []string) []MediaResult { + if !strings.Contains(content, "MEDIA:") || len(roots) == 0 { return nil } matches := mediaPathPattern.FindAllString(content, -1) @@ -158,7 +199,7 @@ func extractMediaFromContent(content, workspace string) []MediaResult { if path == "" { continue } - cleaned, ok := confineToWorkspace(path, workspace) + cleaned, ok := confineToAnyRoot(path, roots) if !ok { continue } diff --git a/internal/agent/loop_media_test.go b/internal/agent/loop_media_test.go index c0e1d542..9943a048 100644 --- a/internal/agent/loop_media_test.go +++ b/internal/agent/loop_media_test.go @@ -1,9 +1,13 @@ package agent import ( + "context" "os" "path/filepath" + "slices" "testing" + + "github.com/nextlevelbuilder/goclaw/internal/tools" ) // writeTempFile drops a zero-byte file at workspace/relPath, creating dirs. @@ -31,6 +35,12 @@ func TestExtractMediaFromContent(t *testing.T) { audioA := writeTempFile(t, ws, "a.mp3") audioB := writeTempFile(t, ws, "b.mp3") chartPath := writeTempFile(t, ws, "charts/q4.png") + partnerRaw := t.TempDir() + partner, err := filepath.EvalSymlinks(partnerRaw) + if err != nil { + t.Fatal(err) + } + partnerPath := writeTempFile(t, partner, "partner-result.png") // Outside-workspace file: should be rejected by containment check. outsideDir := t.TempDir() @@ -38,6 +48,14 @@ func TestExtractMediaFromContent(t *testing.T) { if err := os.WriteFile(outsidePath, nil, 0o644); err != nil { t.Fatal(err) } + hardlinkPath := filepath.Join(partner, "hardlink-result.png") + if err := os.Link(outsidePath, hardlinkPath); err != nil { + t.Skipf("hardlinks not supported: %v", err) + } + symlinkRoot := filepath.Join(t.TempDir(), "delegate") + if err := os.Symlink(outsideDir, symlinkRoot); err != nil { + t.Skipf("symlinks not supported: %v", err) + } // Symlink inside workspace pointing to outside: must be rejected by // EvalSymlinks-then-Rel containment. Covers the P0 ancestor-symlink @@ -59,7 +77,7 @@ func TestExtractMediaFromContent(t *testing.T) { tests := []struct { name string content string - workspace string + roots []string wantPaths []string }{ { @@ -73,54 +91,75 @@ func TestExtractMediaFromContent(t *testing.T) { { name: "relative path resolved + exists", content: "MEDIA:deliver/report.pdf", - workspace: ws, + roots: []string{ws}, wantPaths: []string{reportPath}, }, { name: "multiple tokens deduped", content: "First: MEDIA:a.mp3\nSecond: MEDIA:b.mp3\nAgain: MEDIA:a.mp3", - workspace: ws, + roots: []string{ws}, wantPaths: []string{audioA, audioB}, }, { name: "markdown wrapped and punctuation stripped", content: `![chart](MEDIA:charts/q4.png). See "MEDIA:deliver/report.pdf".`, - workspace: ws, + roots: []string{ws}, wantPaths: []string{chartPath, reportPath}, }, { - name: "hallucinated path dropped (file missing)", - content: "MEDIA:not-real.pdf", - workspace: ws, + name: "hallucinated path dropped (file missing)", + content: "MEDIA:not-real.pdf", + roots: []string{ws}, }, { - name: "path traversal escape blocked", - content: "MEDIA:../leak.pdf", - workspace: ws, + name: "path traversal escape blocked", + content: "MEDIA:../leak.pdf", + roots: []string{ws}, }, { - name: "absolute path outside workspace blocked", - content: "MEDIA:" + outsidePath, - workspace: ws, + name: "absolute path outside workspace blocked", + content: "MEDIA:" + outsidePath, + roots: []string{ws}, + }, + { + name: "absolute collaboration path allowed", + content: "MEDIA:" + partnerPath, + roots: []string{ws, partner}, + wantPaths: []string{partnerPath}, + }, + { + name: "hardlink in collaboration path blocked", + content: "MEDIA:" + hardlinkPath, + roots: []string{ws, partner}, + }, + { + name: "symlinked collaboration root blocked", + content: "MEDIA:" + outsidePath, + roots: []string{ws, symlinkRoot}, + }, + { + name: "relative path does not search collaboration roots", + content: "MEDIA:partner-result.png", + roots: []string{ws, partner}, }, { name: "absolute path with no workspace dropped", content: "MEDIA:" + reportPath, }, { - name: "symlink leaf rejected by Lstat", - content: "MEDIA:shortcut-to-leak.pdf", - workspace: ws, + name: "symlink leaf rejected by Lstat", + content: "MEDIA:shortcut-to-leak.pdf", + roots: []string{ws}, }, { - name: "ancestor symlink escape blocked (P0)", - content: "MEDIA:shared/victim.pdf", - workspace: ws, + name: "ancestor symlink escape blocked (P0)", + content: "MEDIA:shared/victim.pdf", + roots: []string{ws}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := extractMediaFromContent(tt.content, tt.workspace) + got := extractMediaFromContent(tt.content, tt.roots) if len(got) != len(tt.wantPaths) { t.Fatalf("count = %d, want %d; got=%+v", len(got), len(tt.wantPaths), got) } @@ -133,6 +172,25 @@ func TestExtractMediaFromContent(t *testing.T) { } } +func TestMediaEgressRoots(t *testing.T) { + ctx := tools.WithToolWorkspace(context.Background(), "/workspace/parent") + ctx = tools.WithToolTeamWorkspace(ctx, "/workspace/team/chat") + ctx = tools.WithToolTeamRoot(ctx, "/workspace/team") + ctx = tools.WithTenantAllowedPaths(ctx, []string{"/workspace/tenant-export"}) + loop := NewLoop(LoopConfig{}) + + got := loop.mediaEgressRoots(ctx) + want := []string{ + "/workspace/parent", + "/workspace/team/chat", + "/workspace/team", + "/workspace/tenant-export", + } + if !slices.Equal(got, want) { + t.Fatalf("mediaEgressRoots() = %v, want %v", got, want) + } +} + // TestConfineToWorkspace exercises the shared media path-containment boundary // directly. It is the single guard that both feeders of MediaResult.Path rely // on, so a regression here would reopen the outbound-exfiltration hole (H2). @@ -180,6 +238,7 @@ func TestConfineToWorkspace(t *testing.T) { {name: "absolute outside workspace rejected", path: outsidePath, workspace: ws, wantOK: false}, {name: "traversal escape rejected", path: "../secret.txt", workspace: ws, wantOK: false}, {name: "missing file rejected", path: "nope.pdf", workspace: ws, wantOK: false}, + {name: "directory rejected", path: "deliver", workspace: ws, wantOK: false}, {name: "empty workspace rejected", path: insidePath, workspace: "", wantOK: false}, {name: "empty path rejected", path: "", workspace: ws, wantOK: false}, {name: "leaf symlink rejected", path: "shortcut.txt", workspace: ws, wantOK: false, symlink: true}, diff --git a/internal/agent/loop_pipeline_adapter.go b/internal/agent/loop_pipeline_adapter.go index 24957378..2673da79 100644 --- a/internal/agent/loop_pipeline_adapter.go +++ b/internal/agent/loop_pipeline_adapter.go @@ -45,7 +45,7 @@ func (l *Loop) runViaPipeline(ctx context.Context, req RunRequest) (*RunResult, if err != nil { return nil, err } - return convertRunResult(pResult), nil + return redactDelegationRunResult(&req, convertRunResult(pResult)), nil } // buildPipelineDeps maps Loop fields + methods to PipelineDeps callbacks. @@ -94,7 +94,7 @@ func (l *Loop) buildPipelineDeps(req *RunRequest, bridgeRS *runState) pipeline.P ResolveContextWindow: l.resolveEffectiveContextWindow, EmitEvent: func(event any) { if ae, ok := event.(AgentEvent); ok { - l.emit(ae) + l.emit(redactDelegationAgentEvent(req, ae)) } }, diff --git a/internal/agent/loop_pipeline_callbacks.go b/internal/agent/loop_pipeline_callbacks.go index c20af8f7..6c9e0773 100644 --- a/internal/agent/loop_pipeline_callbacks.go +++ b/internal/agent/loop_pipeline_callbacks.go @@ -40,7 +40,7 @@ func (l *Loop) pipelineCallbacks(req *RunRequest, bridgeRS *runState) pipelineCa event.ChatID = req.ChatID event.SessionKey = req.SessionKey event.TenantID = l.tenantID - l.emit(event) + l.emit(redactDelegationAgentEvent(req, event)) } return pipelineCallbackSet{ emitRun: emitRun, @@ -188,7 +188,7 @@ func (l *Loop) makeEnrichMedia(req *RunRequest) func(ctx context.Context, state if len(msgs) == 0 { return nil } - enrichedCtx, enrichedMsgs, _ := l.enrichInputMedia(ctx, req, msgs) + enrichedCtx, enrichedMsgs, currentRefs := l.enrichInputMedia(ctx, req, msgs) // Propagate enriched context (media images/docs/audio/video refs for tools). state.Ctx = enrichedCtx // Update history with enriched messages (media tags, inline images). @@ -196,6 +196,23 @@ func (l *Loop) makeEnrichMedia(req *RunRequest) func(ctx context.Context, state if len(enrichedMsgs) > 1 { state.Messages.SetHistory(enrichedMsgs[1:]) } + // Preserve the enriched current input for the first session checkpoint. + // Inline image bytes stay request-local; durable history stores only the + // logical tags plus absolute MediaRefs used internally for exact lookup. + if len(currentRefs) > 0 { + for i := len(enrichedMsgs) - 1; i >= 0; i-- { + if enrichedMsgs[i].Role != "user" { + continue + } + req.enrichedInputMessage = providers.Message{ + Role: "user", + Content: enrichedMsgs[i].Content, + MediaRefs: append([]providers.MediaRef(nil), currentRefs...), + } + req.hasEnrichedInputMessage = true + break + } + } return nil } } @@ -365,6 +382,12 @@ func (l *Loop) makeCallLLM(req *RunRequest, emitRun func(AgentEvent)) func(ctx c chatReq.Options[providers.OptPeerKind] = req.PeerKind chatReq.Options[providers.OptLocalKey] = req.LocalKey chatReq.Options[providers.OptWorkspace] = tools.ToolWorkspaceFromCtx(ctx) + if delegationID := tools.DelegationIDFromCtx(ctx); delegationID != "" { + chatReq.Options[providers.OptDelegationID] = delegationID + } + if inputs := tools.DelegationArtifactInputsFromCtx(ctx); inputs != "" { + chatReq.Options[providers.OptDelegationInputs] = inputs + } // Pass the policy-filtered allowed tool set so the Claude CLI provider // can restrict its native built-in tools (Bash, Edit, Write, Read, // WebFetch, WebSearch) to what the agent's tool policy actually allows. @@ -706,13 +729,17 @@ func (l *Loop) makeFlushMessages(req *RunRequest) func(ctx context.Context, sess return func(ctx context.Context, sessionKey string, msgs []providers.Message) error { if !userMsgFlushed && !req.HideInput && req.Message != "" { userMsgFlushed = true - l.sessions.AddMessage(ctx, sessionKey, providers.Message{ + inputMessage := providers.Message{ Role: "user", Content: req.Message, - }) + } + if req.hasEnrichedInputMessage { + inputMessage = req.enrichedInputMessage + } + l.sessions.AddMessage(ctx, sessionKey, redactDelegationMessage(req, inputMessage)) } for _, msg := range msgs { - l.sessions.AddMessage(ctx, sessionKey, msg) + l.sessions.AddMessage(ctx, sessionKey, redactDelegationMessage(req, msg)) } return nil } diff --git a/internal/agent/loop_pipeline_callbacks_test.go b/internal/agent/loop_pipeline_callbacks_test.go index 95a49e39..8f4e0765 100644 --- a/internal/agent/loop_pipeline_callbacks_test.go +++ b/internal/agent/loop_pipeline_callbacks_test.go @@ -2,13 +2,17 @@ package agent import ( "context" + "os" + "path/filepath" "strings" "testing" "github.com/google/uuid" + "github.com/nextlevelbuilder/goclaw/internal/bus" "github.com/nextlevelbuilder/goclaw/internal/pipeline" "github.com/nextlevelbuilder/goclaw/internal/providers" + "github.com/nextlevelbuilder/goclaw/internal/tools" "github.com/nextlevelbuilder/goclaw/pkg/protocol" ) @@ -25,6 +29,32 @@ func (p finalThinkingStreamProvider) ChatStream(context.Context, providers.ChatR func (p finalThinkingStreamProvider) DefaultModel() string { return "test-model" } func (p finalThinkingStreamProvider) Name() string { return "test-provider" } +type requestCaptureProvider struct { + request providers.ChatRequest +} + +func (p *requestCaptureProvider) Chat(_ context.Context, req providers.ChatRequest) (*providers.ChatResponse, error) { + p.request = req + return &providers.ChatResponse{Content: "ok"}, nil +} + +func (p *requestCaptureProvider) ChatStream(_ context.Context, req providers.ChatRequest, _ func(providers.StreamChunk)) (*providers.ChatResponse, error) { + p.request = req + return &providers.ChatResponse{Content: "ok"}, nil +} + +func (p *requestCaptureProvider) DefaultModel() string { return "test-model" } +func (p *requestCaptureProvider) Name() string { return "test-provider" } + +type recordingSessionStore struct { + *nopSessionStore + added []providers.Message +} + +func (s *recordingSessionStore) AddMessage(_ context.Context, _ string, msg providers.Message) { + s.added = append(s.added, msg) +} + func TestMakeCallLLM_StreamsFinalThinkingWhenNoThinkingChunkArrives(t *testing.T) { col := &eventCollector{} loop := &Loop{id: "test-agent", onEvent: col.onEvent} @@ -58,6 +88,73 @@ func TestMakeCallLLM_StreamsFinalThinkingWhenNoThinkingChunkArrives(t *testing.T } } +func TestMakeCallLLMPropagatesDelegationArtifactBridgeOptions(t *testing.T) { + provider := &requestCaptureProvider{} + loop := &Loop{id: "target-agent", agentUUID: uuid.New()} + req := &RunRequest{RunID: "run-1", SessionKey: "delegate:session"} + state := &pipeline.RunState{Provider: provider, Model: "test-model"} + ctx := tools.WithToolWorkspace(context.Background(), "/runtime/outputs") + ctx = tools.WithDelegationID(ctx, "delegation-id") + ctx = tools.WithDelegationArtifactInputs(ctx, "/runtime/inputs") + + if _, err := loop.makeCallLLM(req, func(AgentEvent) {})(ctx, state, providers.ChatRequest{}); err != nil { + t.Fatal(err) + } + if provider.request.Options[providers.OptDelegationID] != "delegation-id" || + provider.request.Options[providers.OptDelegationInputs] != "/runtime/inputs" { + t.Fatalf("delegation options = %#v", provider.request.Options) + } +} + +func TestEnrichedInputMediaPersistsForNextTurn(t *testing.T) { + sourceDir := t.TempDir() + sourcePath := filepath.Join(sourceDir, "photo.png") + if err := os.WriteFile(sourcePath, minimalPNG, 0o600); err != nil { + t.Fatal(err) + } + workspace := t.TempDir() + sessions := &recordingSessionStore{nopSessionStore: &nopSessionStore{}} + loop := &Loop{sessions: sessions} + req := &RunRequest{ + SessionKey: "session-media", + Message: ``, + Media: []bus.MediaFile{{ + Path: sourcePath, + MimeType: "image/png", + Filename: "photo.png", + }}, + } + state := &pipeline.RunState{Messages: pipeline.NewMessageBuffer(providers.Message{Role: "system"})} + state.Messages.SetHistory([]providers.Message{{Role: "user", Content: req.Message}}) + ctx := tools.WithToolWorkspace(context.Background(), workspace) + + if err := loop.makeEnrichMedia(req)(ctx, state); err != nil { + t.Fatal(err) + } + if err := loop.makeFlushMessages(req)(ctx, req.SessionKey, nil); err != nil { + t.Fatal(err) + } + + if len(sessions.added) != 1 { + t.Fatalf("persisted messages = %d, want 1", len(sessions.added)) + } + persisted := sessions.added[0] + if len(persisted.MediaRefs) != 1 { + t.Fatalf("persisted MediaRefs = %#v, want one image ref", persisted.MediaRefs) + } + if strings.Contains(persisted.Content, workspace) { + t.Fatalf("persisted content leaked workspace path: %q", persisted.Content) + } + if !strings.Contains(persisted.Content, `path=".uploads/`) { + t.Fatalf("persisted content lacks logical image path: %q", persisted.Content) + } + + nextTurnRefs := collectRefsByKind([]providers.Message{persisted}, nil, "image") + if len(nextTurnRefs) != 1 || nextTurnRefs[0].ID != persisted.MediaRefs[0].ID { + t.Fatalf("next-turn refs = %#v, want persisted exact ID", nextTurnRefs) + } +} + func TestPromptCacheOptionsHelpers(t *testing.T) { tenantID := uuid.MustParse("11111111-1111-1111-1111-111111111111") agentID := uuid.MustParse("22222222-2222-2222-2222-222222222222") diff --git a/internal/agent/loop_pipeline_tool_callbacks.go b/internal/agent/loop_pipeline_tool_callbacks.go index 60280c11..a79bd26e 100644 --- a/internal/agent/loop_pipeline_tool_callbacks.go +++ b/internal/agent/loop_pipeline_tool_callbacks.go @@ -384,6 +384,6 @@ func makeToolEmitRun(l *Loop, req *RunRequest) func(AgentEvent) { event.Channel = req.Channel event.ChatID = req.ChatID event.TenantID = l.tenantID - l.emit(event) + l.emit(redactDelegationAgentEvent(req, event)) } } diff --git a/internal/agent/loop_pipeline_tool_callbacks_test.go b/internal/agent/loop_pipeline_tool_callbacks_test.go index ce4b432a..faf519d2 100644 --- a/internal/agent/loop_pipeline_tool_callbacks_test.go +++ b/internal/agent/loop_pipeline_tool_callbacks_test.go @@ -2,9 +2,11 @@ package agent import ( "context" + "strings" "sync" "testing" + "github.com/nextlevelbuilder/goclaw/internal/bus" "github.com/nextlevelbuilder/goclaw/internal/config" "github.com/nextlevelbuilder/goclaw/internal/pipeline" "github.com/nextlevelbuilder/goclaw/internal/providers" @@ -72,6 +74,44 @@ func newTestLoopForToolCallbacks(onEvent func(AgentEvent)) *Loop { } } +func TestProcessToolResultSuppressesUnpublishedArtifactMediaBeforeEvent(t *testing.T) { + col := &eventCollector{} + loop := newTestLoopForToolCallbacks(col.onEvent) + req := &RunRequest{RunID: "run-artifact"} + ctx := tools.WithDelegationID(context.Background(), "delegation-1") + ctx = tools.WithDelegationArtifactInputs(ctx, "/runtime/delegations/delegation-1/inputs") + result := &tools.Result{ + ForLLM: "created image\nMEDIA:/runtime/delegations/delegation-1/outputs/image.png", + Media: []bus.MediaFile{{Path: "/runtime/delegations/delegation-1/outputs/image.png"}}, + } + + toolMsg, _, _ := loop.processToolResult( + ctx, + &runState{}, + req, + col.onEvent, + providers.ToolCall{ID: "tc-artifact", Name: "create_image"}, + "create_image", + result, + false, + ) + + if strings.Contains(toolMsg.Content, "MEDIA:") || len(result.Media) != 0 { + t.Fatalf("unpublished media survived result policy: message=%q media=%#v", toolMsg.Content, result.Media) + } + events := col.filter(protocol.AgentEventToolResult) + if len(events) != 1 { + t.Fatalf("tool result events = %d, want 1", len(events)) + } + payload, ok := events[0].Payload.(map[string]any) + if !ok { + t.Fatalf("event payload type = %T", events[0].Payload) + } + if value, _ := payload["result"].(string); strings.Contains(value, "MEDIA:") { + t.Fatalf("event leaked unpublished media marker: %q", value) + } +} + // TestMakeExecuteToolCall_EmitsToolCallEvent verifies the sequential wrapper // emits a tool.call event before running tool I/O. func TestMakeExecuteToolCall_EmitsToolCallEvent(t *testing.T) { diff --git a/internal/agent/loop_run.go b/internal/agent/loop_run.go index 7837db29..5137adde 100644 --- a/internal/agent/loop_run.go +++ b/internal/agent/loop_run.go @@ -18,6 +18,7 @@ import ( func (l *Loop) Run(ctx context.Context, req RunRequest) (*RunResult, error) { l.activeRuns.Add(1) defer l.activeRuns.Add(-1) + ctx = withDelegationArtifactTextRedactor(ctx, &req) // Per-run emit wrapper: enriches every AgentEvent with delegation + routing context. emitRun := func(event AgentEvent) { @@ -32,7 +33,7 @@ func (l *Loop) Run(ctx context.Context, req RunRequest) (*RunResult, error) { event.ChatID = req.ChatID event.SessionKey = req.SessionKey event.TenantID = store.TenantIDFromContext(ctx) - l.emit(event) + l.emit(redactDelegationAgentEvent(&req, event)) } emitRun(AgentEvent{ @@ -75,7 +76,7 @@ func (l *Loop) Run(ctx context.Context, req RunRequest) (*RunResult, error) { UserID: req.UserID, Channel: req.Channel, Name: traceName, - InputPreview: truncateStr(req.Message, l.traceCollector.PreviewMaxLen()), + InputPreview: tracing.RedactText(ctx, truncateStr(req.Message, l.traceCollector.PreviewMaxLen())), Status: store.TraceStatusRunning, StartTime: now, CreatedAt: now, @@ -140,7 +141,7 @@ func (l *Loop) Run(ctx context.Context, req RunRequest) (*RunResult, error) { l.emitAgentSpanEnd(safeCtx, agentSpanID, runStart, nil, context.Canceled) } l.traceCollector.FinishTrace(safeCtx, traceID, store.TraceStatusError, - "trace finalized by safety net (likely panic or goroutine leak)", "") + tracing.RedactText(safeCtx, "trace finalized by safety net (likely panic or goroutine leak)"), "") }() } @@ -195,7 +196,7 @@ func (l *Loop) Run(ctx context.Context, req RunRequest) (*RunResult, error) { traceCtx = context.WithoutCancel(ctx) traceStatus = store.TraceStatusCancelled } - l.traceCollector.FinishTrace(traceCtx, traceID, traceStatus, err.Error(), "") + l.traceCollector.FinishTrace(traceCtx, traceID, traceStatus, tracing.RedactText(traceCtx, err.Error()), "") } return nil, err } @@ -243,7 +244,8 @@ func (l *Loop) Run(ctx context.Context, req RunRequest) (*RunResult, error) { if !isChildTrace && l.traceCollector != nil && traceID != uuid.Nil { traceFinalized = true if result != nil { - l.traceCollector.FinishTrace(ctx, traceID, store.TraceStatusCompleted, "", truncateStr(result.Content, l.traceCollector.PreviewMaxLen())) + l.traceCollector.FinishTrace(ctx, traceID, store.TraceStatusCompleted, "", + tracing.RedactText(ctx, truncateStr(result.Content, l.traceCollector.PreviewMaxLen()))) } else { l.traceCollector.FinishTrace(ctx, traceID, store.TraceStatusCompleted, "", "") } diff --git a/internal/agent/loop_tools.go b/internal/agent/loop_tools.go index 3fd43ae4..3f04fa71 100644 --- a/internal/agent/loop_tools.go +++ b/internal/agent/loop_tools.go @@ -36,6 +36,12 @@ func (l *Loop) processToolResult( hadBootstrap bool, ) (toolMsg providers.Message, warningMsgs []providers.Message, action toolResultAction) { + // Agent Link outputs are not valid outbound media until the runtime has + // validated and atomically published the delegation manifest. Apply this + // before logs, events, loop detection, or message construction so no + // pre-publication MEDIA marker escapes through observability surfaces. + tools.ApplyDelegationArtifactResultPolicy(ctx, result) + // Record for loop detection. argsHash := rs.loopDetector.record(registryName, tc.Arguments) rs.loopDetector.recordResult(argsHash, result.ForLLM) @@ -85,6 +91,7 @@ func (l *Loop) processToolResult( // Collect MEDIA: paths from tool results. // Prefer result.Media (explicit) over ForLLM MEDIA: prefix (legacy) to avoid duplicates. + mediaRoots := l.mediaEgressRoots(ctx) if len(result.Media) > 0 { // Egress containment: a tool that sets result.Media[].Path to a path // outside every allowed scope (e.g. /etc/passwd from a prompt-injected @@ -94,10 +101,6 @@ func (l *Loop) processToolResult( // to — agent workspace, team workspace, and tenant-allowed paths — so a // cross-workspace file (e.g. a teammate-produced file in the shared team // workspace, or a synchronous delegatee's output) is not wrongly dropped. - mediaRoots := append([]string{ - tools.ToolWorkspaceFromCtx(ctx), - tools.ToolTeamWorkspaceFromCtx(ctx), - }, l.tenantAllowedPaths...) for i, mf := range result.Media { cleaned, ok := confineToAnyRoot(mf.Path, mediaRoots) if !ok { @@ -123,7 +126,7 @@ func (l *Loop) processToolResult( // Telegram sendDocument). A malicious or buggy tool emitting // MEDIA:/etc/passwd is dropped here — fixing every channel at the source // rather than per-channel. Mirrors extractMediaFromContent containment. - if cleaned, ok := confineToWorkspace(mr.Path, tools.ToolWorkspaceFromCtx(ctx)); ok { + if cleaned, ok := confineToAnyRoot(mr.Path, mediaRoots); ok { mr.Path = cleaned rs.mediaResults = append(rs.mediaResults, *mr) } else { diff --git a/internal/agent/loop_tracing.go b/internal/agent/loop_tracing.go index f9ee428b..1607521e 100644 --- a/internal/agent/loop_tracing.go +++ b/internal/agent/loop_tracing.go @@ -152,7 +152,7 @@ func (l *Loop) emitLLMSpanStart(ctx context.Context, start time.Time, iteration } } - collector.EmitSpan(span) + collector.EmitSpan(tracing.RedactSpan(ctx, span)) return spanID } @@ -244,7 +244,7 @@ func (l *Loop) emitLLMSpanEnd(ctx context.Context, spanID uuid.UUID, start time. updates["metadata"] = spanMetadata } - collector.EmitSpanUpdate(spanID, traceID, updates) + collector.EmitSpanUpdate(spanID, traceID, tracing.RedactSpanUpdates(ctx, updates)) } // --------------------------------------------------------------------------- @@ -289,7 +289,7 @@ func (l *Loop) emitToolSpanStart(ctx context.Context, start time.Time, toolName, span.TenantID = store.MasterTenantID } - collector.EmitSpan(span) + collector.EmitSpan(tracing.RedactSpan(ctx, span)) return spanID } @@ -353,7 +353,7 @@ func (l *Loop) emitToolSpanEnd(ctx context.Context, spanID uuid.UUID, start time } } - collector.EmitSpanUpdate(spanID, traceID, updates) + collector.EmitSpanUpdate(spanID, traceID, tracing.RedactSpanUpdates(ctx, updates)) } // --------------------------------------------------------------------------- @@ -401,7 +401,7 @@ func (l *Loop) emitAgentSpanStart(ctx context.Context, agentSpanID uuid.UUID, st span.TenantID = store.MasterTenantID } - collector.EmitSpan(span) + collector.EmitSpan(tracing.RedactSpan(ctx, span)) } // emitAgentSpanEnd finalizes the running root agent span with results. @@ -433,7 +433,7 @@ func (l *Loop) emitAgentSpanEnd(ctx context.Context, agentSpanID uuid.UUID, star // with child spans that directly report model usage. } - collector.EmitSpanUpdate(agentSpanID, traceID, updates) + collector.EmitSpanUpdate(agentSpanID, traceID, tracing.RedactSpanUpdates(ctx, updates)) } // previewLimitForVerbose returns the preview character limit based on verbose mode. diff --git a/internal/agent/loop_types.go b/internal/agent/loop_types.go index c2c43449..7d9a52e0 100644 --- a/internal/agent/loop_types.go +++ b/internal/agent/loop_types.go @@ -666,11 +666,13 @@ type RunRequest struct { OnTraceCreated func(traceID uuid.UUID) // Delegation context (set when running as a delegate agent) - DelegationID string // delegation ID for event correlation - TeamID string // team ID (if delegation is team-scoped) - TeamTaskID string // team task ID (if delegation has an associated task) - ParentAgentID string // parent agent key that initiated the delegation - LeaderAgentID string // leader agent UUID for member memory read fallback + DelegationID string // delegation ID for event correlation + DelegateInputsPath string // runtime-only read-only staged inputs root + DelegateOutputsPath string // runtime-only writable exchange workspace + TeamID string // team ID (if delegation is team-scoped) + TeamTaskID string // team task ID (if delegation has an associated task) + ParentAgentID string // parent agent key that initiated the delegation + LeaderAgentID string // leader agent UUID for member memory read fallback // Workspace scope propagation (set by delegation, read by workspace tools) WorkspaceChannel string @@ -678,6 +680,12 @@ type RunRequest struct { // TeamWorkspace overrides the member agent's workspace with the team's workspace // so file operations (read/write/image/audio) use the shared team directory. TeamWorkspace string + + // enrichedInputMessage is populated by the media stage and consumed by the + // first persistence checkpoint. It keeps current-turn MediaRefs and logical + // tags durable without storing inline image bytes. + enrichedInputMessage providers.Message + hasEnrichedInputMessage bool } // RunResult is the output of a completed agent run. diff --git a/internal/agent/media.go b/internal/agent/media.go index de97ed95..daa2067d 100644 --- a/internal/agent/media.go +++ b/internal/agent/media.go @@ -8,6 +8,7 @@ import ( "log/slog" "os" "path/filepath" + "strconv" "strings" "github.com/google/uuid" @@ -299,13 +300,23 @@ func copyMediaFile(src, dst string) error { return out.Close() } -// enrichDocumentPaths updates the last user message to include persisted file paths -// in tags. This allows skills (e.g. pdf skill via exec) to access -// the file directly, matching how Claude Code skills work with file paths. -func (l *Loop) enrichDocumentPaths(messages []providers.Message, refs []providers.MediaRef) { +// enrichDocumentPaths updates document tags with exact media IDs and +// workspace-relative paths. It upgrades historical tags and pairs current refs +// with the last user message. Paths outside the active workspace are omitted. +func (l *Loop) enrichDocumentPaths(messages []providers.Message, refs []providers.MediaRef, workspace string) { if len(messages) == 0 { return } + + // Upgrade historical tags as they re-enter the prompt. Current-turn refs + // are handled below because they are not yet attached to message history. + for i := range messages { + if messages[i].Role != "user" || len(messages[i].MediaRefs) == 0 { + continue + } + messages[i].Content = l.enrichDocumentTagContent(messages[i].Content, messages[i].MediaRefs, workspace) + } + lastIdx := -1 for i := len(messages) - 1; i >= 0; i-- { if messages[i].Role == "user" { @@ -316,41 +327,62 @@ func (l *Loop) enrichDocumentPaths(messages []providers.Message, refs []provider if lastIdx < 0 { return } + messages[lastIdx].Content = l.enrichDocumentTagContent(messages[lastIdx].Content, refs, workspace) +} - content := messages[lastIdx].Content +func (l *Loop) enrichDocumentTagContent(content string, refs []providers.MediaRef, workspace string) string { for _, ref := range refs { if ref.Kind != "document" { continue } p := ref.Path if p == "" && l.mediaStore != nil { - var err error - p, err = l.mediaStore.LoadPath(ref.ID) - if err != nil { - continue + if loaded, err := l.mediaStore.LoadPath(ref.ID); err == nil { + p = loaded } } - if p == "" { + logical := logicalWorkspaceMediaPath(workspace, p) + + updateTag := func(tag string) string { + tag = setTagAttr(tag, "id", ref.ID) + if logical == "" { + return removeTagAttr(tag, "path") + } + return setTagAttr(tag, "path", logical) + } + + // Prefer a tag already carrying the exact media ID. This also upgrades + // legacy absolute paths without relying on attribute order. + var replaced bool + content, replaced = replaceFirstMediaTag(content, " without a path — covers bare, named, and file= variants. + // Fallback: pair the next tag without an ID with this persisted ref. content, _ = replaceFirstMediaTag(content, " and tags so the LLM can reference them. -func (l *Loop) enrichAudioIDs(messages []providers.Message, refs []providers.MediaRef) { +// enrichAudioIDs updates audio/voice tags with exact media IDs and logical +// workspace paths. Historical tags are upgraded when they re-enter the prompt. +func (l *Loop) enrichAudioIDs(messages []providers.Message, refs []providers.MediaRef, workspace string) { if len(messages) == 0 { return } + + for i := range messages { + if messages[i].Role != "user" || len(messages[i].MediaRefs) == 0 { + continue + } + messages[i].Content = l.enrichAudioTagContent(messages[i].Content, messages[i].MediaRefs, workspace) + } + lastIdx := -1 for i := len(messages) - 1; i >= 0; i-- { if messages[i].Role == "user" { @@ -361,31 +393,55 @@ func (l *Loop) enrichAudioIDs(messages []providers.Message, refs []providers.Med if lastIdx < 0 { return } + messages[lastIdx].Content = l.enrichAudioTagContent(messages[lastIdx].Content, refs, workspace) +} - content := messages[lastIdx].Content +func (l *Loop) enrichAudioTagContent(content string, refs []providers.MediaRef, workspace string) string { for _, ref := range refs { if ref.Kind != "audio" { continue } - idAttr := fmt.Sprintf(" id=%q", ref.ID) + p := ref.Path + if p == "" && l.mediaStore != nil { + if loaded, err := l.mediaStore.LoadPath(ref.ID); err == nil { + p = loaded + } + } + logical := logicalWorkspaceMediaPath(workspace, p) + updateTag := func(tag string) string { + tag = setTagAttr(tag, "id", ref.ID) + if logical == "" { + return removeTagAttr(tag, "path") + } + return setTagAttr(tag, "path", logical) + } + // Upgrade a historical tag already carrying this exact ID first. var replaced bool content, replaced = replaceFirstMediaTag(content, " tags so the LLM knows images were received -// and stored. The path attribute allows tools called via MCP bridge (e.g. +// and logical workspace paths in tags so the LLM knows images +// were received and stored. The path attribute allows tools called via MCP bridge (e.g. // claude-cli) to access images via read_image(path=...) even though the // bridge context does not carry WithMediaImages. -func (l *Loop) enrichImageIDs(messages []providers.Message, refs []providers.MediaRef) { +func (l *Loop) enrichImageIDs(messages []providers.Message, refs []providers.MediaRef, workspace string) { if len(messages) == 0 { return } @@ -448,8 +504,8 @@ func (l *Loop) enrichImageIDs(messages []providers.Message, refs []providers.Med } idAttr := fmt.Sprintf(" id=%q", ref.ID) pathAttr := "" - if ref.Path != "" { - pathAttr = fmt.Sprintf(" path=%q", ref.Path) + if logical := logicalWorkspaceMediaPath(workspace, ref.Path); logical != "" { + pathAttr = fmt.Sprintf(" path=%q", logical) } content, _ = replaceFirstMediaTag(content, " tags. This enables the LLM to call read_image(path=...) // to analyze images without inline base64 (saving context tokens). // Unlike enrichImageIDs (last user message only), this enriches ALL messages // so historical images from prior turns are also accessible via file path. -func (l *Loop) enrichImagePaths(messages []providers.Message) { - if l.mediaStore == nil { - return - } +func (l *Loop) enrichImagePaths(messages []providers.Message, workspace string) { for i := range messages { if messages[i].Role != "user" || len(messages[i].MediaRefs) == 0 { continue @@ -485,7 +538,7 @@ func (l *Loop) enrichImagePaths(messages []providers.Message) { continue } p := ref.Path - if p == "" { + if p == "" && l.mediaStore != nil { var err error p, err = l.mediaStore.LoadPath(ref.ID) if err != nil { @@ -495,14 +548,26 @@ func (l *Loop) enrichImagePaths(messages []providers.Message) { if p == "" { continue } - pathAttr := fmt.Sprintf(" path=%q", p) + logical := logicalWorkspaceMediaPath(workspace, p) + if logical == "" { + var stripped bool + content, stripped = replaceFirstMediaTag(content, "") + strings.Join(attrs, "") + ">" } +func setTagAttr(tag, attr, value string) string { + prefix := " " + attr + `="` + start := strings.Index(tag, prefix) + if start < 0 { + return appendTagAttrs(tag, fmt.Sprintf(` %s=%q`, attr, value)) + } + valueStart := start + len(prefix) + valueEnd := strings.IndexByte(tag[valueStart:], '"') + if valueEnd < 0 { + return tag + } + valueEnd += valueStart + quoted := strconv.Quote(value) + return tag[:valueStart] + quoted[1:len(quoted)-1] + tag[valueEnd:] +} + +func removeTagAttr(tag, attr string) string { + prefix := " " + attr + `="` + start := strings.Index(tag, prefix) + if start < 0 { + return tag + } + valueStart := start + len(prefix) + valueEnd := strings.IndexByte(tag[valueStart:], '"') + if valueEnd < 0 { + return tag + } + return tag[:start] + tag[valueStart+valueEnd+1:] +} + // maxMediaReloadMessages is the default number of recent messages with image MediaRefs // to reload for LLM vision context. const maxMediaReloadMessages = 5 diff --git a/internal/agent/media_test.go b/internal/agent/media_test.go index f5fd4144..ecc985d4 100644 --- a/internal/agent/media_test.go +++ b/internal/agent/media_test.go @@ -1,13 +1,16 @@ package agent import ( + "context" "os" "path/filepath" "strings" "testing" + "github.com/nextlevelbuilder/goclaw/internal/bus" "github.com/nextlevelbuilder/goclaw/internal/media" "github.com/nextlevelbuilder/goclaw/internal/providers" + "github.com/nextlevelbuilder/goclaw/internal/tools" ) // TestEnrichImageIDs_BareTag verifies enrichment of a bare tag @@ -20,15 +23,62 @@ func TestEnrichImageIDs_BareTag(t *testing.T) { refs := []providers.MediaRef{{ID: "img-1", Kind: "image", Path: "/tmp/a.jpg"}} var loop Loop - loop.enrichImageIDs(messages, refs) + loop.enrichImageIDs(messages, refs, "/tmp") got := messages[0].Content - want := `check ` + want := `check ` if got != want { t.Fatalf("bare tag enrichment:\n got %q\nwant %q", got, want) } } +func TestEnrichInputMedia_DelegateRehomesCallerAttachment(t *testing.T) { + callerWorkspace := t.TempDir() + delegateWorkspace := t.TempDir() + source := filepath.Join(callerWorkspace, "reference.jpg") + if err := os.WriteFile(source, []byte("reference"), 0644); err != nil { + t.Fatal(err) + } + + req := &RunRequest{ + SessionKey: "delegate:parent:cppai-media:test", + RunKind: "delegate", + Media: []bus.MediaFile{{ + Path: source, + MimeType: "image/jpeg", + Filename: "reference.jpg", + }}, + } + messages := []providers.Message{{ + Role: "user", + Content: "Use this exact reference: " + source, + }} + ctx := tools.WithToolWorkspace(context.Background(), delegateWorkspace) + + loop := Loop{id: "cppai-media"} + _, gotMessages, refs := loop.enrichInputMedia(ctx, req, messages) + + if len(refs) != 1 { + t.Fatalf("refs = %#v, want one delegate-owned attachment", refs) + } + if !strings.HasPrefix(refs[0].Path, filepath.Join(delegateWorkspace, ".uploads")+string(filepath.Separator)) { + t.Fatalf("persisted path = %q, want under delegate workspace %q", refs[0].Path, delegateWorkspace) + } + if strings.Contains(gotMessages[0].Content, source) { + t.Fatalf("delegate message still exposes caller path: %q", gotMessages[0].Content) + } + logicalPath := logicalWorkspaceMediaPath(delegateWorkspace, refs[0].Path) + if logicalPath == "" || !strings.Contains(gotMessages[0].Content, logicalPath) { + t.Fatalf("delegate message = %q, want logical copied path %q", gotMessages[0].Content, logicalPath) + } + if strings.Contains(gotMessages[0].Content, delegateWorkspace) { + t.Fatalf("delegate message leaked delegate workspace: %q", gotMessages[0].Content) + } + if !strings.Contains(gotMessages[0].Content, `` messages := []providers.Message{{ Role: "user", @@ -126,10 +176,69 @@ func TestEnrichImagePaths_NoDoubleEnrich(t *testing.T) { }} loop := Loop{mediaStore: testMediaStore(t)} - loop.enrichImagePaths(messages) + loop.enrichImagePaths(messages, "/workspace") - if messages[0].Content != original { - t.Fatalf("double-enrichment detected:\n got %q\nwant %q", messages[0].Content, original) + want := `` + if messages[0].Content != want { + t.Fatalf("legacy path was not upgraded:\n got %q\nwant %q", messages[0].Content, want) + } +} + +func TestEnrichImagePathsEscapesLogicalPathAttribute(t *testing.T) { + workspace := t.TempDir() + mediaPath := filepath.Join(workspace, ".uploads", `photo"quoted.jpg`) + messages := []providers.Message{{ + Role: "user", + Content: ``, + MediaRefs: []providers.MediaRef{{ + ID: "img-1", + Kind: "image", + Path: mediaPath, + }}, + }} + + var loop Loop + loop.enrichImagePaths(messages, workspace) + + want := `` + if messages[0].Content != want { + t.Fatalf("escaped path = %q, want %q", messages[0].Content, want) + } +} + +func TestLogicalWorkspaceMediaPathRejectsOutsideWorkspace(t *testing.T) { + workspace := filepath.Join(t.TempDir(), "agent") + inside := filepath.Join(workspace, ".uploads", "photo.jpg") + outside := filepath.Join(filepath.Dir(workspace), "other-agent", ".uploads", "photo.jpg") + + if got := logicalWorkspaceMediaPath(workspace, inside); got != ".uploads/photo.jpg" { + t.Fatalf("inside path = %q, want %q", got, ".uploads/photo.jpg") + } + if got := logicalWorkspaceMediaPath(workspace, outside); got != "" { + t.Fatalf("outside path = %q, want omitted", got) + } + if got := logicalWorkspaceMediaPath(workspace, "../escape.jpg"); got != "" { + t.Fatalf("traversal path = %q, want omitted", got) + } +} + +func TestEnrichImagePathsStripsOutsideWorkspacePath(t *testing.T) { + messages := []providers.Message{{ + Role: "user", + Content: ``, + MediaRefs: []providers.MediaRef{{ + ID: "img-1", + Kind: "image", + Path: "/workspace/other-agent/.uploads/photo.jpg", + }}, + }} + + var loop Loop + loop.enrichImagePaths(messages, "/workspace/current-agent") + + want := `` + if messages[0].Content != want { + t.Fatalf("outside workspace path was not stripped: got %q, want %q", messages[0].Content, want) } } @@ -148,10 +257,10 @@ func TestEnrichImagePaths_AttributeOrderIndependence(t *testing.T) { }} loop := Loop{mediaStore: testMediaStore(t)} - loop.enrichImagePaths(messages) + loop.enrichImagePaths(messages, "/workspace") got := messages[0].Content - if !strings.Contains(got, `path="/workspace/.uploads/img-1.jpg"`) { + if !strings.Contains(got, `path=".uploads/img-1.jpg"`) { t.Fatalf("expected path to be added regardless of attribute order, got %q", got) } if !strings.Contains(got, `url="https://cdn.example.com/photo.jpg"`) { @@ -173,9 +282,9 @@ func TestEnrichImageIDs_MultipleRefs(t *testing.T) { } var loop Loop - loop.enrichImageIDs(messages, refs) + loop.enrichImageIDs(messages, refs, "/tmp") - want := `first ` + "\n" + `second ` + want := `first ` + "\n" + `second ` if messages[0].Content != want { t.Fatalf("multi-ref alignment:\n got %q\nwant %q", messages[0].Content, want) } @@ -223,53 +332,129 @@ func TestEnrichImagePaths_MultipleRefsKeepTagAlignment(t *testing.T) { var loop Loop loop.mediaStore = mediaStore - loop.enrichImagePaths(messages) + loop.enrichImagePaths(messages, "/persisted") - want := `first ` + - "\n" + `second ` + want := `first ` + + "\n" + `second ` if messages[0].Content != want { t.Fatalf("enrichImagePaths() content = %q, want %q", messages[0].Content, want) } } func TestEnrichDocumentPaths_MultipleRefs(t *testing.T) { + workspace := t.TempDir() messages := []providers.Message{{ Role: "user", Content: "first \nsecond ", }} refs := []providers.MediaRef{ - {ID: "doc-a", Kind: "document", Path: "/tmp/a.pdf"}, - {ID: "doc-b", Kind: "document", Path: "/tmp/b.pdf"}, + {ID: "doc-a", Kind: "document", Path: filepath.Join(workspace, ".uploads", "a.pdf")}, + {ID: "doc-b", Kind: "document", Path: filepath.Join(workspace, ".uploads", "b.pdf")}, } var loop Loop - loop.enrichDocumentPaths(messages, refs) + loop.enrichDocumentPaths(messages, refs, workspace) - want := `first ` + "\n" + `second ` + want := `first ` + "\n" + + `second ` if messages[0].Content != want { t.Fatalf("multi-ref alignment:\n got %q\nwant %q", messages[0].Content, want) } } +func TestEnrichDocumentPathsStripsOutsideWorkspacePath(t *testing.T) { + workspace := filepath.Join(t.TempDir(), "current-agent") + outside := filepath.Join(filepath.Dir(workspace), "other-agent", ".uploads", "secret.pdf") + messages := []providers.Message{{ + Role: "user", + Content: ``, + }} + refs := []providers.MediaRef{{ + ID: "doc-1", + Kind: "document", + Path: outside, + }} + + var loop Loop + loop.enrichDocumentPaths(messages, refs, workspace) + + want := `` + if messages[0].Content != want { + t.Fatalf("outside workspace path was not stripped: got %q, want %q", messages[0].Content, want) + } +} + +func TestEnrichDocumentPathsUpgradesHistoricalAbsolutePath(t *testing.T) { + workspace := t.TempDir() + docPath := filepath.Join(workspace, ".uploads", "history.pdf") + messages := []providers.Message{ + { + Role: "user", + Content: ``, + MediaRefs: []providers.MediaRef{{ + ID: "doc-old", + Kind: "document", + Path: docPath, + }}, + }, + {Role: "user", Content: "follow up"}, + } + + var loop Loop + loop.enrichDocumentPaths(messages, nil, workspace) + + want := `` + if messages[0].Content != want { + t.Fatalf("historical path was not upgraded: got %q, want %q", messages[0].Content, want) + } +} + func TestEnrichAudioIDs_MultipleRefs(t *testing.T) { + workspace := t.TempDir() messages := []providers.Message{{ Role: "user", Content: "first \nsecond ", }} refs := []providers.MediaRef{ - {ID: "aud-a", Kind: "audio"}, - {ID: "aud-b", Kind: "audio"}, + {ID: "aud-a", Kind: "audio", Path: filepath.Join(workspace, ".uploads", "a.mp3")}, + {ID: "aud-b", Kind: "audio", Path: filepath.Join(workspace, ".uploads", "b.mp3")}, } var loop Loop - loop.enrichAudioIDs(messages, refs) + loop.enrichAudioIDs(messages, refs, workspace) - want := `first ` + "\n" + `second ` + want := `first ` + "\n" + + `second ` if messages[0].Content != want { t.Fatalf("multi-ref alignment:\n got %q\nwant %q", messages[0].Content, want) } } +func TestEnrichAudioIDsUpgradesHistoricalLogicalPath(t *testing.T) { + workspace := t.TempDir() + audioPath := filepath.Join(workspace, ".uploads", "history.mp3") + messages := []providers.Message{ + { + Role: "user", + Content: ``, + MediaRefs: []providers.MediaRef{{ + ID: "aud-old", + Kind: "audio", + Path: audioPath, + }}, + }, + {Role: "user", Content: "follow up"}, + } + + var loop Loop + loop.enrichAudioIDs(messages, nil, workspace) + + want := `` + if messages[0].Content != want { + t.Fatalf("historical audio path was not upgraded: got %q, want %q", messages[0].Content, want) + } +} + func TestEnrichVideoIDs_MultipleRefs(t *testing.T) { messages := []providers.Message{{ Role: "user", @@ -301,9 +486,9 @@ func TestEnrichImageIDs_MoreRefsThanTags(t *testing.T) { } var loop Loop - loop.enrichImageIDs(messages, refs) + loop.enrichImageIDs(messages, refs, "/tmp") - want := `only one ` + want := `only one ` if messages[0].Content != want { t.Fatalf("more refs than tags:\n got %q\nwant %q", messages[0].Content, want) } @@ -320,9 +505,9 @@ func TestEnrichImageIDs_MoreTagsThanRefs(t *testing.T) { } var loop Loop - loop.enrichImageIDs(messages, refs) + loop.enrichImageIDs(messages, refs, "/tmp") - want := `first ` + "\n" + `second ` + want := `first ` + "\n" + `second ` if messages[0].Content != want { t.Fatalf("more tags than refs:\n got %q\nwant %q", messages[0].Content, want) } @@ -341,7 +526,7 @@ func TestEnrichAudioIDs_MixedAudioAndVoice(t *testing.T) { } var loop Loop - loop.enrichAudioIDs(messages, refs) + loop.enrichAudioIDs(messages, refs, "") want := `hear this ` + "\n" + `and this ` if messages[0].Content != want { diff --git a/internal/agent/prompt_builder_orchestration_test.go b/internal/agent/prompt_builder_orchestration_test.go index e19f43d2..0025b0c3 100644 --- a/internal/agent/prompt_builder_orchestration_test.go +++ b/internal/agent/prompt_builder_orchestration_test.go @@ -111,4 +111,10 @@ func TestBridgePromptBuilder_OrchestrationTargetContent(t *testing.T) { if !strings.Contains(output, "Does work") { t.Error("output should contain description") } + if !strings.Contains(output, "Return generated files or media artifacts to the caller") { + t.Error("output should tell delegates to return media artifacts through the caller") + } + if !strings.Contains(output, "Do not ask the delegatee to send directly to your current chat") { + t.Error("output should prevent direct delivery attempts from the delegate channel") + } } diff --git a/internal/agent/prompt_config_types.go b/internal/agent/prompt_config_types.go index 4a08fabb..0a201c9c 100644 --- a/internal/agent/prompt_config_types.go +++ b/internal/agent/prompt_config_types.go @@ -98,8 +98,6 @@ type WorkspaceSectionData struct { ActivePath string Scope string Enforced bool - ReadOnlyPaths []string - SharedPath *string ContextFiles []string EnforcementMsg string } diff --git a/internal/agent/systemprompt.go b/internal/agent/systemprompt.go index 48a6ff27..6d8f7e67 100644 --- a/internal/agent/systemprompt.go +++ b/internal/agent/systemprompt.go @@ -207,7 +207,7 @@ var coreToolSummaries = map[string]string{ "session_status": "Show session status (model, tokens, compaction count)", "sessions_history": "Fetch message history for a session", "sessions_send": "Send a message into another session", - "read_image": "Analyze images — call with path from tags, or a direct HTTP/HTTPS URL via the 'url' parameter", + "read_image": "Analyze images — use the exact logical path from tags; media_id and direct HTTP/HTTPS URLs are also supported", "read_audio": "Analyze audio — call with media_id from tags", "read_video": "Analyze video — call with media_id from tags, or a direct HTTP/HTTPS URL via the 'url' parameter", "create_video": "Generate videos from text descriptions using AI", @@ -673,7 +673,7 @@ func buildToolingSection(toolNames []string, hasSandbox bool, shellDenyGroups ma lines = append(lines, "", "### Media Files", - `When users send media (, , , ), use the corresponding read_* tool with the path/media_id. For archives (.zip, .tar.gz, etc.), use exec with the document path to inspect/extract the archive.`, + `When users send media (, , , ), use the corresponding read_* tool with the exact path/media_id from the tag. Never reconstruct or guess an absolute workspace path. For archives (.zip, .tar.gz, etc.), use exec with the document path to inspect/extract the archive.`, "You have full vision/audio/video capabilities. NEVER say you cannot see images or files.", ) } diff --git a/internal/agent/systemprompt_sections.go b/internal/agent/systemprompt_sections.go index e6390fe4..fb7a8ef6 100644 --- a/internal/agent/systemprompt_sections.go +++ b/internal/agent/systemprompt_sections.go @@ -771,6 +771,8 @@ func buildOrchestrationSection(data OrchestrationSectionData) []string { lines = append(lines, "", "Use `delegate` with the agent_key of the target agent. Do NOT invent agent keys.", + "Return generated files or media artifacts to the caller through the delegate result.", + "Do not ask the delegatee to send directly to your current chat; delegated runs use an internal `delegate` channel and the caller handles final delivery.", "", ) return lines diff --git a/internal/channels/telegram/commands_subagents.go b/internal/channels/telegram/commands_subagents.go index 6bd885cc..b23704d5 100644 --- a/internal/channels/telegram/commands_subagents.go +++ b/internal/channels/telegram/commands_subagents.go @@ -18,6 +18,10 @@ const maxSubagentsInList = 30 // subagentStatusIcon returns an icon for each subagent task status. func subagentStatusIcon(status string) string { switch status { + case "queued": + return "⏳" + case "waiting_child": + return "↪️" case "completed": return "✅" case "failed": @@ -52,18 +56,20 @@ func (c *Channel) handleSubagentsList(ctx context.Context, chatID int64, isGroup return } - agentKey := c.AgentID() - if agentKey == "" { - send("Subagent tasks are not available (no agent configured).") + rootAgentID, err := c.resolveAgentUUID(ctx) + if err != nil { + slog.Warn("subagents command: resolve agent UUID failed", "error", err) + send("Subagent tasks are not available (agent could not be resolved).") return } - tasks, err := c.subagentTaskStore.ListByParent(ctx, agentKey, "") + tasks, err := c.subagentTaskStore.ListByParent(ctx, rootAgentID, "") if err != nil { slog.Warn("subagents command: ListByParent failed", "error", err) send("Failed to list subagent tasks. Please try again.") return } + tasks = filterSelfCloneTasks(tasks) if len(tasks) == 0 { send("No subagent tasks found.") @@ -142,7 +148,13 @@ func (c *Channel) handleSubagentDetail(ctx context.Context, chatID int64, text s return } - task, err := c.subagentTaskStore.Get(ctx, taskID) + rootAgentID, err := c.resolveAgentUUID(ctx) + if err != nil { + slog.Warn("subagent command: resolve agent UUID failed", "error", err) + send("Subagent tasks are not available (agent could not be resolved).") + return + } + task, err := c.subagentTaskStore.Get(ctx, rootAgentID, taskID) if err != nil { slog.Warn("subagent command: Get failed", "id", idArg, "error", err) send("Failed to load subagent task. Please try again.") @@ -152,6 +164,10 @@ func (c *Channel) handleSubagentDetail(ctx context.Context, chatID int64, text s send(fmt.Sprintf("Task %q not found. Use /subagents to see available tasks.", idArg[:8])) return } + if isDelegationCompletion(task) { + send(fmt.Sprintf("Task %q is a delegation result; retrieve it with the delegate tool.", idArg[:8])) + return + } send(formatSubagentDetail(task)) } @@ -181,7 +197,13 @@ func (c *Channel) handleSubagentCallback(ctx context.Context, query *telego.Call return } - task, err := c.subagentTaskStore.Get(ctx, taskID) + rootAgentID, err := c.resolveAgentUUID(ctx) + if err != nil { + slog.Warn("subagent callback: resolve agent UUID failed", "error", err) + send("Subagent tasks are not available (agent could not be resolved).") + return + } + task, err := c.subagentTaskStore.Get(ctx, rootAgentID, taskID) if err != nil { slog.Warn("subagent callback: Get failed", "id", taskIDStr, "error", err) send("Failed to load subagent task.") @@ -191,6 +213,10 @@ func (c *Channel) handleSubagentCallback(ctx context.Context, query *telego.Call send(fmt.Sprintf("Task %s not found.", taskIDStr[:8])) return } + if isDelegationCompletion(task) { + send("This task is a delegation result and is not part of /subagents.") + return + } send(formatSubagentDetail(task)) } @@ -218,3 +244,21 @@ func formatSubagentDetail(t *store.SubagentTaskData) string { } return sb.String() } + +func filterSelfCloneTasks(tasks []store.SubagentTaskData) []store.SubagentTaskData { + filtered := tasks[:0] + for i := range tasks { + if !isDelegationCompletion(&tasks[i]) { + filtered = append(filtered, tasks[i]) + } + } + return filtered +} + +func isDelegationCompletion(task *store.SubagentTaskData) bool { + if task == nil || task.Metadata == nil { + return false + } + kind, _ := task.Metadata["completion_kind"].(string) + return kind == "delegate" +} diff --git a/internal/childrun/admission.go b/internal/childrun/admission.go new file mode 100644 index 00000000..a434b2d3 --- /dev/null +++ b/internal/childrun/admission.go @@ -0,0 +1,901 @@ +package childrun + +import ( + "context" + "errors" + "fmt" + "sync" + "sync/atomic" + + "github.com/google/uuid" +) + +var ( + ErrChildRunBusy = errors.New("child_run_busy") + ErrChildRunClosed = errors.New("child_run_closed") + ErrChildRunDrainTimeout = errors.New("child_run_drain_timeout") + ErrChildRunInvalidContinuation = errors.New("child_run_invalid_continuation") + ErrChildRunContinuationDepth = errors.New("child_run_continuation_depth") +) + +const maxChildRunContinuationDepth = 64 + +type ChildRunState string + +const ( + ChildRunNew ChildRunState = "new" + ChildRunQueued ChildRunState = "queued" + ChildRunRunning ChildRunState = "running" + ChildRunWaitingChild ChildRunState = "waiting_child" + ChildRunCompleted ChildRunState = "completed" + ChildRunFailed ChildRunState = "failed" + ChildRunCancelled ChildRunState = "cancelled" +) + +// ChildRunConstraints are rechecked atomically whenever a run or suspended +// continuation is granted. Nil root identity means process-limit-only work. +// +// TaskID, Depth, and MaxDepth are optional for top-level runs, but TaskID and +// direct-child lineage are required for synchronous continuation handoff. +type ChildRunConstraints struct { + TenantID uuid.UUID + RootAgentID uuid.UUID + RootLimit int + TaskID string + ParentTaskID string + ParentFanout int + Depth int + MaxDepth int +} + +type ChildRunStats struct { + Active int + Pending int + Live int + Closed bool +} + +type childRunRoot struct { + tenant uuid.UUID + agent uuid.UUID +} + +type childRunParent struct { + root childRunRoot + parent string +} + +// ChildRunAdmission owns process-wide child execution admission. It creates +// one dispatcher goroutine and stops it after Close has fully drained. +type ChildRunAdmission struct { + mu sync.Mutex + processLimit int + pendingLimit int + nextSequence uint64 + active int + live int + pendingIndependent int + pending []*childRunRequest + running map[*ChildRunTicket]*childRunFrame + liveTickets map[*ChildRunTicket]struct{} + rootActive map[childRunRoot]int + parentActive map[childRunParent]int + closed bool + wake chan struct{} + stop chan struct{} + dispatcherDone chan struct{} + drained chan struct{} + drainedOnce sync.Once + stopOnce sync.Once +} + +// ChildRunTicket represents one independent admission chain. Nested +// synchronous continuations reuse this chain and its original FIFO sequence. +type ChildRunTicket struct { + controller *ChildRunAdmission + ctx context.Context + sequence uint64 + maxDepth int + state ChildRunState + err error + + activated bool + cancelRequested bool + finished bool + live bool + started chan struct{} + done chan struct{} + doneOnce sync.Once + stopCancel func() bool + + top *childRunFrame + current *childRunFrame + stack []*childRunFrame + requests map[*childRunRequest]struct{} + topRequest *childRunRequest +} + +// ChildRunLease is controller-issued execution authority. Continue validates +// that the lease is the chain's current owner before it can hand capacity to a +// direct synchronous child. +type ChildRunLease struct { + controller *ChildRunAdmission + ticket *ChildRunTicket + frame *childRunFrame + continueMu sync.Mutex + releaseRequested atomic.Bool +} + +// ContinuationParent exposes immutable structural lineage for a direct +// synchronous continuation. Agent-level spawn policy is enforced by the tool +// that owns the child; this lineage only protects admission handoff integrity. +func (l *ChildRunLease) ContinuationParent() (taskID string, depth int, ok bool) { + if l == nil || l.frame == nil || l.frame.constraints.TaskID == "" { + return "", 0, false + } + return l.frame.constraints.TaskID, l.frame.constraints.Depth, true +} + +type childRunFrame struct { + constraints ChildRunConstraints + ctx context.Context + cancel context.CancelFunc + run func(context.Context, *ChildRunLease) + lease *ChildRunLease + state ChildRunState + cancelRequested bool + stopCancel func() bool +} + +type childRunRequest struct { + ticket *ChildRunTicket + frame *childRunFrame + sequence uint64 + independent bool + launch bool + activated bool + granted chan struct{} + cancelled chan struct{} + cancelErr error + resolved bool +} + +func NewChildRunAdmission(processLimit, pendingLimit int) *ChildRunAdmission { + if processLimit < 1 { + processLimit = 1 + } + if pendingLimit < 0 { + pendingLimit = 0 + } + c := &ChildRunAdmission{ + processLimit: processLimit, + pendingLimit: pendingLimit, + running: make(map[*ChildRunTicket]*childRunFrame), + liveTickets: make(map[*ChildRunTicket]struct{}), + rootActive: make(map[childRunRoot]int), + parentActive: make(map[childRunParent]int), + wake: make(chan struct{}, 1), + stop: make(chan struct{}), + dispatcherDone: make(chan struct{}), + drained: make(chan struct{}), + } + go c.dispatch() + return c +} + +// Enqueue reserves one bounded independent pending record but leaves it +// dormant. Activate makes the run eligible after its owner installs visible +// queued task state. +func (c *ChildRunAdmission) Enqueue( + ctx context.Context, + constraints ChildRunConstraints, + run func(context.Context, *ChildRunLease), +) (*ChildRunTicket, error) { + if run == nil { + return nil, fmt.Errorf("child run callback is required") + } + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return nil, err + } + + c.mu.Lock() + defer c.mu.Unlock() + if c.closed { + return nil, ErrChildRunClosed + } + if c.pendingIndependent >= c.pendingLimit { + return nil, ErrChildRunBusy + } + maxDepth := constraints.MaxDepth + if maxDepth <= 0 || maxDepth > maxChildRunContinuationDepth { + maxDepth = maxChildRunContinuationDepth + } + if constraints.Depth < 0 || constraints.Depth > maxDepth { + return nil, ErrChildRunContinuationDepth + } + constraints.MaxDepth = maxDepth + + c.nextSequence++ + ticket := &ChildRunTicket{ + controller: c, + ctx: ctx, + sequence: c.nextSequence, + maxDepth: maxDepth, + state: ChildRunNew, + started: make(chan struct{}), + done: make(chan struct{}), + requests: make(map[*childRunRequest]struct{}), + } + frame := c.newFrameLocked(ticket, ctx, constraints, run) + request := &childRunRequest{ + ticket: ticket, + frame: frame, + sequence: ticket.sequence, + independent: true, + launch: true, + granted: make(chan struct{}), + cancelled: make(chan struct{}), + } + ticket.top = frame + ticket.stack = []*childRunFrame{frame} + ticket.topRequest = request + ticket.requests[request] = struct{}{} + c.pending = append(c.pending, request) + c.pendingIndependent++ + ticket.stopCancel = context.AfterFunc(ctx, func() { + ticket.Cancel() + }) + return ticket, nil +} + +func (t *ChildRunTicket) Activate() error { + c := t.controller + c.mu.Lock() + defer c.mu.Unlock() + + if t.finished || t.cancelRequested { + if t.err != nil { + return t.err + } + return context.Canceled + } + if c.closed { + c.cancelTicketLocked(t, ErrChildRunClosed) + return ErrChildRunClosed + } + if t.state != ChildRunNew { + return nil + } + t.activated = true + t.state = ChildRunQueued + t.top.state = ChildRunQueued + t.topRequest.activated = true + c.signalLocked() + return nil +} + +// Cancel decides cancel-versus-grant while holding the controller lock. +// Running work is marked cancelled immediately, but its capacity is retained +// until the callback actually returns. +func (t *ChildRunTicket) Cancel() bool { + c := t.controller + c.mu.Lock() + defer c.mu.Unlock() + if t.finished || t.cancelRequested { + return false + } + c.cancelTicketLocked(t, context.Canceled) + return true +} + +func (t *ChildRunTicket) State() ChildRunState { + t.controller.mu.Lock() + defer t.controller.mu.Unlock() + return t.state +} + +func (t *ChildRunTicket) Err() error { + t.controller.mu.Lock() + defer t.controller.mu.Unlock() + return t.err +} + +func (t *ChildRunTicket) Started() <-chan struct{} { return t.started } +func (t *ChildRunTicket) Done() <-chan struct{} { return t.done } + +// Release returns execution capacity exactly once while keeping the ticket's +// lifecycle barrier open until the callback returns. Controller-owned wrappers +// call it only after execution and immutable result capture, before persistence +// and announcement side effects. +func (l *ChildRunLease) Release() { + if l == nil || l.controller == nil || l.ticket == nil || l.frame == nil { + return + } + c := l.controller + c.mu.Lock() + defer c.mu.Unlock() + if !l.releaseRequested.CompareAndSwap(false, true) { + return + } + if c.running[l.ticket] == l.frame && l.ticket.current == l.frame { + c.releaseActiveLocked(l.ticket, l.frame) + } +} + +// Continue synchronously transfers this lease's execution capacity to a direct +// child and reacquires the parent's original capacity before returning. The +// child executes on the caller's goroutine; no dispatcher execution goroutine +// is created for synchronous nesting. +func (l *ChildRunLease) Continue( + ctx context.Context, + constraints ChildRunConstraints, + run func(context.Context, *ChildRunLease), +) error { + if l == nil || l.controller == nil || l.ticket == nil || l.frame == nil { + return ErrChildRunInvalidContinuation + } + if run == nil { + return fmt.Errorf("child run callback is required") + } + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } + + // One executing parent owns one transferable continuation. Concurrent + // synchronous calls serialize here; async children use Enqueue instead. + l.continueMu.Lock() + defer l.continueMu.Unlock() + + c := l.controller + c.mu.Lock() + constraints, err := c.validateContinuationLocked(l, constraints) + if err != nil { + c.mu.Unlock() + return err + } + + parent := l.frame + child := c.newFrameLocked(l.ticket, ctx, constraints, run) + l.ticket.stack = append(l.ticket.stack, child) + parent.state = ChildRunWaitingChild + l.ticket.state = ChildRunWaitingChild + c.releaseActiveLocked(l.ticket, parent) + + request := c.newContinuationRequestLocked(l.ticket, child) + sameRoot := sameRootConstraint(parent.constraints, child.constraints) + if sameRoot && c.eligibleLocked(child.constraints) { + c.grantLocked(request) + } else { + c.pending = append(c.pending, request) + c.signalLocked() + } + c.mu.Unlock() + + if err := waitChildRunRequest(request); err != nil { + resume := c.finishContinuation(child, err) + if resume != nil { + if resumeErr := waitChildRunRequest(resume); resumeErr != nil { + return resumeErr + } + } + return err + } + + childErr := invokeChildRun(child) + if childErr == nil && child.ctx.Err() != nil { + childErr = child.ctx.Err() + } + resume := c.finishContinuation(child, childErr) + if resume != nil { + if err := waitChildRunRequest(resume); err != nil { + return err + } + } + return childErr +} + +func (c *ChildRunAdmission) Stats() ChildRunStats { + c.mu.Lock() + defer c.mu.Unlock() + return ChildRunStats{ + Active: c.active, + Pending: len(c.pending), + Live: c.live, + Closed: c.closed, + } +} + +// Close is idempotent. The first call closes intake and cancels queued and +// active work. A timeout leaves the controller and dispatcher available for a +// later drain retry; the dispatcher stops only after every callback returns. +func (c *ChildRunAdmission) Close(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() + } + + c.mu.Lock() + if !c.closed { + c.closed = true + tickets := make(map[*ChildRunTicket]struct{}) + for _, request := range c.pending { + tickets[request.ticket] = struct{}{} + } + for ticket := range c.running { + tickets[ticket] = struct{}{} + } + for ticket := range c.liveTickets { + tickets[ticket] = struct{}{} + } + for ticket := range tickets { + c.cancelTicketLocked(ticket, ErrChildRunClosed) + } + c.maybeDrainedLocked() + c.signalLocked() + } + drained := c.drained + c.mu.Unlock() + + select { + case <-drained: + <-c.dispatcherDone + return nil + case <-ctx.Done(): + return fmt.Errorf("%w: %v", ErrChildRunDrainTimeout, ctx.Err()) + } +} + +func (c *ChildRunAdmission) dispatch() { + defer close(c.dispatcherDone) + for { + select { + case <-c.wake: + c.dispatchAvailable() + case <-c.stop: + return + } + } +} + +func (c *ChildRunAdmission) dispatchAvailable() { + for { + c.mu.Lock() + request := c.oldestEligibleLocked() + if request == nil { + c.maybeDrainedLocked() + c.mu.Unlock() + return + } + launch := request.launch + ticket := request.ticket + frame := request.frame + granted := c.grantLocked(request) + c.mu.Unlock() + + if launch && granted { + go c.executeTop(ticket, frame) + } + } +} + +func (c *ChildRunAdmission) oldestEligibleLocked() *childRunRequest { + if c.closed || c.active >= c.processLimit { + return nil + } + var oldest *childRunRequest + for _, request := range c.pending { + if request.resolved || !request.activated { + continue + } + if request.frame.ctx.Err() != nil || !c.eligibleLocked(request.frame.constraints) { + continue + } + if oldest == nil || request.sequence < oldest.sequence { + oldest = request + } + } + return oldest +} + +func (c *ChildRunAdmission) eligibleLocked(constraints ChildRunConstraints) bool { + if c.active >= c.processLimit { + return false + } + root := childRunRoot{tenant: constraints.TenantID, agent: constraints.RootAgentID} + if constraints.RootAgentID != uuid.Nil && constraints.RootLimit > 0 && + c.rootActive[root] >= constraints.RootLimit { + return false + } + parent := childRunParent{root: root, parent: constraints.ParentTaskID} + return constraints.ParentTaskID == "" || constraints.ParentFanout <= 0 || + c.parentActive[parent] < constraints.ParentFanout +} + +func (c *ChildRunAdmission) grantLocked(request *childRunRequest) bool { + if request.resolved { + return false + } + if err := request.frame.ctx.Err(); err != nil { + c.cancelRequestLocked(request, err) + return false + } + c.removePendingLocked(request) + request.resolved = true + request.frame.state = ChildRunRunning + request.ticket.current = request.frame + c.running[request.ticket] = request.frame + c.incrementLocked(request.frame.constraints) + + if request.launch { + request.ticket.state = ChildRunRunning + request.ticket.live = true + c.live++ + c.liveTickets[request.ticket] = struct{}{} + close(request.ticket.started) + } else if request.frame == request.ticket.top { + request.ticket.state = ChildRunRunning + } else { + request.ticket.state = ChildRunWaitingChild + } + close(request.granted) + return true +} + +func (c *ChildRunAdmission) incrementLocked(constraints ChildRunConstraints) { + c.active++ + root := childRunRoot{tenant: constraints.TenantID, agent: constraints.RootAgentID} + if constraints.RootAgentID != uuid.Nil { + c.rootActive[root]++ + } + if constraints.ParentTaskID != "" { + c.parentActive[childRunParent{root: root, parent: constraints.ParentTaskID}]++ + } +} + +func (c *ChildRunAdmission) releaseActiveLocked(ticket *ChildRunTicket, frame *childRunFrame) { + if c.running[ticket] != frame { + return + } + delete(c.running, ticket) + ticket.current = nil + c.active-- + + root := childRunRoot{ + tenant: frame.constraints.TenantID, + agent: frame.constraints.RootAgentID, + } + if frame.constraints.RootAgentID != uuid.Nil { + decrementChildRunCount(c.rootActive, root) + } + if frame.constraints.ParentTaskID != "" { + decrementChildRunCount( + c.parentActive, + childRunParent{root: root, parent: frame.constraints.ParentTaskID}, + ) + } + c.signalLocked() +} + +func (c *ChildRunAdmission) executeTop(ticket *ChildRunTicket, frame *childRunFrame) { + var callbackErr error + defer func() { + if recovered := recover(); recovered != nil { + callbackErr = fmt.Errorf("child run panic: %v", recovered) + } + c.finishTop(ticket, callbackErr) + }() + frame.run(frame.ctx, frame.lease) +} + +func invokeChildRun(frame *childRunFrame) (callbackErr error) { + defer func() { + if recovered := recover(); recovered != nil { + callbackErr = fmt.Errorf("child run panic: %v", recovered) + } + }() + frame.run(frame.ctx, frame.lease) + return callbackErr +} + +func (c *ChildRunAdmission) finishTop(ticket *ChildRunTicket, callbackErr error) { + c.mu.Lock() + defer c.mu.Unlock() + if ticket.finished { + return + } + + if current := c.running[ticket]; current != nil { + c.releaseActiveLocked(ticket, current) + } + for request := range ticket.requests { + c.cancelRequestLocked(request, terminalChildRunError(ticket)) + } + for _, frame := range ticket.stack { + if frame.stopCancel != nil { + frame.stopCancel() + } + frame.cancel() + } + if ticket.stopCancel != nil { + ticket.stopCancel() + } + + switch { + case ticket.cancelRequested: + ticket.state = ChildRunCancelled + if ticket.err == nil { + ticket.err = context.Canceled + } + case callbackErr != nil: + ticket.state = ChildRunFailed + ticket.err = callbackErr + default: + ticket.state = ChildRunCompleted + } + ticket.top.state = ticket.state + ticket.finished = true + if ticket.live { + ticket.live = false + c.live-- + delete(c.liveTickets, ticket) + } + ticket.doneOnce.Do(func() { close(ticket.done) }) + c.maybeDrainedLocked() + c.signalLocked() +} + +func (c *ChildRunAdmission) finishContinuation( + child *childRunFrame, + callbackErr error, +) *childRunRequest { + c.mu.Lock() + defer c.mu.Unlock() + ticket := child.lease.ticket + + if c.running[ticket] == child { + c.releaseActiveLocked(ticket, child) + } + if child.stopCancel != nil { + child.stopCancel() + } + wasCancelled := child.cancelRequested || child.ctx.Err() != nil + contextErr := child.ctx.Err() + child.cancel() + + switch { + case wasCancelled: + child.state = ChildRunCancelled + if callbackErr == nil { + callbackErr = contextErr + } + case callbackErr != nil: + child.state = ChildRunFailed + default: + child.state = ChildRunCompleted + } + + if len(ticket.stack) > 1 && ticket.stack[len(ticket.stack)-1] == child { + ticket.stack = ticket.stack[:len(ticket.stack)-1] + } + if ticket.cancelRequested || ticket.finished || c.closed { + return nil + } + parent := ticket.stack[len(ticket.stack)-1] + if parent.ctx.Err() != nil { + c.cancelTicketLocked(ticket, parent.ctx.Err()) + return nil + } + parent.state = ChildRunQueued + ticket.state = ChildRunWaitingChild + request := c.newContinuationRequestLocked(ticket, parent) + c.pending = append(c.pending, request) + c.signalLocked() + return request +} + +func (c *ChildRunAdmission) validateContinuationLocked( + lease *ChildRunLease, + child ChildRunConstraints, +) (ChildRunConstraints, error) { + ticket := lease.ticket + if c.closed { + return child, ErrChildRunClosed + } + if ticket.finished || ticket.cancelRequested { + return child, terminalChildRunError(ticket) + } + if c.running[ticket] != lease.frame || ticket.current != lease.frame || + lease.frame.state != ChildRunRunning { + return child, ErrChildRunInvalidContinuation + } + parent := lease.frame.constraints + if parent.TaskID == "" || child.TaskID == "" || child.ParentTaskID != parent.TaskID { + return child, ErrChildRunInvalidContinuation + } + if len(ticket.stack) >= maxChildRunContinuationDepth { + return child, ErrChildRunContinuationDepth + } + if child.TenantID != parent.TenantID { + return child, ErrChildRunInvalidContinuation + } + if child.Depth != parent.Depth+1 { + return child, ErrChildRunInvalidContinuation + } + if child.Depth > ticket.maxDepth { + return child, ErrChildRunContinuationDepth + } + child.MaxDepth = ticket.maxDepth + if parent.RootAgentID != uuid.Nil && child.RootAgentID == parent.RootAgentID { + child.RootLimit = parent.RootLimit + } + return child, nil +} + +func (c *ChildRunAdmission) newFrameLocked( + ticket *ChildRunTicket, + ctx context.Context, + constraints ChildRunConstraints, + run func(context.Context, *ChildRunLease), +) *childRunFrame { + runCtx, cancel := context.WithCancel(ctx) + frame := &childRunFrame{ + constraints: constraints, + ctx: runCtx, + cancel: cancel, + run: run, + state: ChildRunNew, + } + frame.lease = &ChildRunLease{controller: c, ticket: ticket, frame: frame} + if ticket.top != nil { + frame.stopCancel = context.AfterFunc(runCtx, func() { + c.cancelContinuationFrame(ticket, frame) + }) + } + return frame +} + +func (c *ChildRunAdmission) newContinuationRequestLocked( + ticket *ChildRunTicket, + frame *childRunFrame, +) *childRunRequest { + request := &childRunRequest{ + ticket: ticket, + frame: frame, + sequence: ticket.sequence, + activated: true, + granted: make(chan struct{}), + cancelled: make(chan struct{}), + } + ticket.requests[request] = struct{}{} + return request +} + +func (c *ChildRunAdmission) cancelContinuationFrame( + ticket *ChildRunTicket, + frame *childRunFrame, +) { + c.mu.Lock() + defer c.mu.Unlock() + if ticket.finished || frame.state == ChildRunCompleted || + frame.state == ChildRunFailed || frame.state == ChildRunCancelled { + return + } + frame.cancelRequested = true + frame.cancel() + for request := range ticket.requests { + if request.frame == frame { + c.cancelRequestLocked(request, frame.ctx.Err()) + } + } + c.signalLocked() +} + +func (c *ChildRunAdmission) cancelTicketLocked(ticket *ChildRunTicket, err error) { + if ticket.finished || ticket.cancelRequested { + return + } + ticket.cancelRequested = true + ticket.err = err + ticket.state = ChildRunCancelled + for _, frame := range ticket.stack { + frame.cancelRequested = true + frame.cancel() + } + for request := range ticket.requests { + c.cancelRequestLocked(request, err) + } + + if !ticket.live { + ticket.finished = true + ticket.top.state = ChildRunCancelled + if ticket.stopCancel != nil { + ticket.stopCancel() + } + ticket.doneOnce.Do(func() { close(ticket.done) }) + } + c.maybeDrainedLocked() + c.signalLocked() +} + +func (c *ChildRunAdmission) cancelRequestLocked(request *childRunRequest, err error) { + if request.resolved { + return + } + c.removePendingLocked(request) + request.resolved = true + request.cancelErr = err + request.frame.cancelRequested = true + request.frame.state = ChildRunCancelled + close(request.cancelled) +} + +func (c *ChildRunAdmission) removePendingLocked(request *childRunRequest) { + for index, pending := range c.pending { + if pending != request { + continue + } + c.pending = append(c.pending[:index], c.pending[index+1:]...) + if request.independent { + c.pendingIndependent-- + } + break + } + delete(request.ticket.requests, request) +} + +func (c *ChildRunAdmission) maybeDrainedLocked() { + if !c.closed || c.active != 0 || c.live != 0 || len(c.pending) != 0 { + return + } + c.drainedOnce.Do(func() { close(c.drained) }) + c.stopOnce.Do(func() { close(c.stop) }) +} + +func (c *ChildRunAdmission) signalLocked() { + select { + case c.wake <- struct{}{}: + default: + } +} + +func waitChildRunRequest(request *childRunRequest) error { + select { + case <-request.granted: + return nil + case <-request.cancelled: + if request.cancelErr != nil { + return request.cancelErr + } + return context.Canceled + } +} + +func terminalChildRunError(ticket *ChildRunTicket) error { + if ticket.err != nil { + return ticket.err + } + if ticket.cancelRequested { + return context.Canceled + } + return ErrChildRunClosed +} + +func sameRootConstraint(left, right ChildRunConstraints) bool { + return left.TenantID == right.TenantID && + left.RootAgentID != uuid.Nil && + left.RootAgentID == right.RootAgentID && + left.RootLimit == right.RootLimit +} + +func decrementChildRunCount[K comparable](counts map[K]int, key K) { + if counts[key] <= 1 { + delete(counts, key) + return + } + counts[key]-- +} diff --git a/internal/childrun/admission_test.go b/internal/childrun/admission_test.go new file mode 100644 index 00000000..b40e68e5 --- /dev/null +++ b/internal/childrun/admission_test.go @@ -0,0 +1,859 @@ +package childrun + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" +) + +const childRunTestTimeout = 3 * time.Second + +func TestChildRunAdmissionStateTransitionsAndDormantActivation(t *testing.T) { + controller := NewChildRunAdmission(1, 2) + closeChildRunController(t, controller) + + blockFirst := make(chan struct{}) + first := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(context.Context, *ChildRunLease) { + <-blockFirst + }) + waitChildRunSignal(t, first.Started(), "first start") + + var ran atomic.Bool + second, err := controller.Enqueue(context.Background(), ChildRunConstraints{}, func(context.Context, *ChildRunLease) { + ran.Store(true) + }) + if err != nil { + t.Fatalf("Enqueue second: %v", err) + } + if got := second.State(); got != ChildRunNew { + t.Fatalf("state before activation = %q, want %q", got, ChildRunNew) + } + if ran.Load() { + t.Fatal("callback started before activation") + } + if err := second.Activate(); err != nil { + t.Fatalf("Activate second: %v", err) + } + if got := second.State(); got != ChildRunQueued { + t.Fatalf("state while capacity is occupied = %q, want %q", got, ChildRunQueued) + } + + close(blockFirst) + waitChildRunSignal(t, second.Started(), "second start") + waitChildRunSignal(t, second.Done(), "second completion") + if got := second.State(); got != ChildRunCompleted { + t.Fatalf("terminal state = %q, want %q", got, ChildRunCompleted) + } + if !ran.Load() { + t.Fatal("callback did not run after activation") + } +} + +func TestChildRunAdmissionEnforcesProcessRootAndParentLimitsAtGrant(t *testing.T) { + controller := NewChildRunAdmission(3, 8) + closeChildRunController(t, controller) + + tenant := uuid.New() + root := uuid.New() + firstRelease := make(chan struct{}) + firstConstraints := ChildRunConstraints{ + TenantID: tenant, RootAgentID: root, RootLimit: 2, + TaskID: "first", ParentTaskID: "parent", ParentFanout: 1, + } + first := enqueueAndActivateChildRun(t, controller, firstConstraints, func(context.Context, *ChildRunLease) { + <-firstRelease + }) + secondConstraints := firstConstraints + secondConstraints.TaskID = "second" + second := enqueueAndActivateChildRun(t, controller, secondConstraints, func(context.Context, *ChildRunLease) {}) + waitChildRunSignal(t, first.Started(), "first sibling start") + assertChildRunNotSignalled(t, second.Started(), "second sibling bypassed immediate-parent fan-out") + + otherRoot := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{ + TenantID: tenant, RootAgentID: uuid.New(), RootLimit: 1, TaskID: "other", + }, func(context.Context, *ChildRunLease) {}) + waitChildRunSignal(t, otherRoot.Done(), "independent root completion") + + close(firstRelease) + waitChildRunSignal(t, second.Done(), "second sibling completion") + if stats := controller.Stats(); stats.Active != 0 { + t.Fatalf("active after completion = %d, want 0", stats.Active) + } +} + +func TestChildRunAdmissionUsesOldestEligibleWithoutHeadOfLineBlocking(t *testing.T) { + controller := NewChildRunAdmission(2, 8) + closeChildRunController(t, controller) + + tenant := uuid.New() + rootA := uuid.New() + rootB := uuid.New() + blockA := make(chan struct{}) + activeA := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{ + TenantID: tenant, RootAgentID: rootA, RootLimit: 1, TaskID: "active-a", + }, func(context.Context, *ChildRunLease) { <-blockA }) + blockedA := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{ + TenantID: tenant, RootAgentID: rootA, RootLimit: 1, TaskID: "blocked-a", + }, func(context.Context, *ChildRunLease) {}) + eligibleB := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{ + TenantID: tenant, RootAgentID: rootB, RootLimit: 1, TaskID: "eligible-b", + }, func(context.Context, *ChildRunLease) {}) + + waitChildRunSignal(t, activeA.Started(), "active root A start") + waitChildRunSignal(t, eligibleB.Done(), "eligible root B completion") + assertChildRunNotSignalled(t, blockedA.Started(), "root-blocked entry started early") + close(blockA) + waitChildRunSignal(t, blockedA.Done(), "blocked root A completion") +} + +func TestChildRunAdmissionPreservesFIFOAmongEligibleRequests(t *testing.T) { + controller := NewChildRunAdmission(1, 4) + closeChildRunController(t, controller) + + activeRelease := make(chan struct{}) + active := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(context.Context, *ChildRunLease) { + <-activeRelease + }) + waitChildRunSignal(t, active.Started(), "active blocker start") + + firstRelease := make(chan struct{}) + first := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(context.Context, *ChildRunLease) { + <-firstRelease + }) + second := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(context.Context, *ChildRunLease) {}) + close(activeRelease) + waitChildRunSignal(t, first.Started(), "oldest eligible start") + assertChildRunNotSignalled(t, second.Started(), "later eligible request overtook oldest request") + close(firstRelease) + waitChildRunSignal(t, second.Done(), "later eligible completion") +} + +func TestChildRunAdmissionEnforcesStandardAndPerRootCapsAcrossRootlessWork(t *testing.T) { + controller := NewChildRunAdmission(32, 40) + closeChildRunController(t, controller) + + tenant := uuid.New() + root := uuid.New() + startedRoot := make(chan struct{}, 21) + startedRootless := make(chan struct{}, 12) + release := make(chan struct{}) + var tickets []*ChildRunTicket + for index := range 21 { + tickets = append(tickets, enqueueAndActivateChildRun(t, controller, ChildRunConstraints{ + TenantID: tenant, RootAgentID: root, RootLimit: 20, + TaskID: "root-task-" + string(rune('a'+index)), + }, func(context.Context, *ChildRunLease) { + startedRoot <- struct{}{} + <-release + })) + } + for index := range 12 { + tickets = append(tickets, enqueueAndActivateChildRun(t, controller, ChildRunConstraints{ + TaskID: "rootless-task-" + string(rune('a'+index)), + }, func(context.Context, *ChildRunLease) { + startedRootless <- struct{}{} + <-release + })) + } + + for range 20 { + waitChildRunSignal(t, startedRoot, "root-scoped admitted callback") + } + for range 12 { + waitChildRunSignal(t, startedRootless, "rootless admitted callback") + } + assertChildRunNotSignalled(t, startedRoot, "twenty-first same-root callback bypassed root cap") + if stats := controller.Stats(); stats.Active != 32 { + t.Fatalf("active at saturation = %d, want Standard cap 32", stats.Active) + } + + close(release) + for _, ticket := range tickets { + waitChildRunSignal(t, ticket.Done(), "saturated callback completion") + } +} + +func TestChildRunAdmissionQueueCapacityIsExactly128IndependentChains(t *testing.T) { + controller := NewChildRunAdmission(1, 128) + closeChildRunController(t, controller) + + activeRelease := make(chan struct{}) + active := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(context.Context, *ChildRunLease) { + <-activeRelease + }) + waitChildRunSignal(t, active.Started(), "queue blocker start") + + queued := make([]*ChildRunTicket, 0, 128) + for index := range 128 { + ticket, err := controller.Enqueue(context.Background(), ChildRunConstraints{}, func(context.Context, *ChildRunLease) {}) + if err != nil { + t.Fatalf("Enqueue pending ticket %d: %v", index, err) + } + queued = append(queued, ticket) + } + if _, err := controller.Enqueue(context.Background(), ChildRunConstraints{}, func(context.Context, *ChildRunLease) {}); !errors.Is(err, ErrChildRunBusy) { + t.Fatalf("129th pending Enqueue error = %v, want ErrChildRunBusy", err) + } + if stats := controller.Stats(); stats.Pending != 128 { + t.Fatalf("pending at queue saturation = %d, want 128", stats.Pending) + } + for _, ticket := range queued { + if !ticket.Cancel() { + t.Fatal("Cancel dormant queued ticket returned false") + } + } + close(activeRelease) +} + +func TestChildRunAdmissionBoundsIndependentQueueButContinuationReusesChain(t *testing.T) { + controller := NewChildRunAdmission(1, 1) + closeChildRunController(t, controller) + + tenant := uuid.New() + root := uuid.New() + var orderMu sync.Mutex + var order []string + var completedChildLease *ChildRunLease + parentStarted := make(chan struct{}) + continueParent := make(chan struct{}) + + parent := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{ + TenantID: tenant, RootAgentID: root, RootLimit: 1, + TaskID: "parent", Depth: 1, MaxDepth: 3, + }, func(ctx context.Context, lease *ChildRunLease) { + close(parentStarted) + <-continueParent + err := lease.Continue(ctx, ChildRunConstraints{ + TenantID: tenant, RootAgentID: root, RootLimit: 0, + TaskID: "child", ParentTaskID: "parent", ParentFanout: 1, + Depth: 2, MaxDepth: 3, + }, func(_ context.Context, childLease *ChildRunLease) { + completedChildLease = childLease + if stats := controller.Stats(); stats.Active != 1 { + t.Errorf("active during in-place child handoff = %d, want 1", stats.Active) + } + controller.mu.Lock() + rootCount := controller.rootActive[childRunRoot{tenant: tenant, agent: root}] + parentCount := controller.parentActive[childRunParent{ + root: childRunRoot{tenant: tenant, agent: root}, + parent: "parent", + }] + controller.mu.Unlock() + if rootCount != 1 || parentCount != 1 { + t.Errorf("handoff counters root=%d parent=%d, want 1/1", rootCount, parentCount) + } + orderMu.Lock() + order = append(order, "child") + orderMu.Unlock() + }) + if err != nil { + t.Errorf("Continue: %v", err) + return + } + orderMu.Lock() + order = append(order, "parent") + orderMu.Unlock() + }) + waitChildRunSignal(t, parentStarted, "parent start") + + independent := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{ + TaskID: "independent", + }, func(context.Context, *ChildRunLease) { + orderMu.Lock() + order = append(order, "independent") + orderMu.Unlock() + }) + if _, err := controller.Enqueue(context.Background(), ChildRunConstraints{}, func(context.Context, *ChildRunLease) {}); !errors.Is(err, ErrChildRunBusy) { + t.Fatalf("second independent pending Enqueue error = %v, want ErrChildRunBusy", err) + } + + close(continueParent) + waitChildRunSignal(t, parent.Done(), "parent continuation completion") + waitChildRunSignal(t, independent.Done(), "independent completion") + if completedChildLease == nil || completedChildLease.frame.state != ChildRunCompleted { + t.Fatalf("normal continuation state = %v, want %q", completedChildLease, ChildRunCompleted) + } + orderMu.Lock() + defer orderMu.Unlock() + want := []string{"child", "parent", "independent"} + if len(order) != len(want) { + t.Fatalf("execution order = %v, want %v", order, want) + } + for index := range want { + if order[index] != want[index] { + t.Fatalf("execution order = %v, want %v", order, want) + } + } +} + +func TestChildRunAdmissionContinuationValidatesLineageAndDepth(t *testing.T) { + controller := NewChildRunAdmission(1, 4) + closeChildRunController(t, controller) + + result := make(chan error, 4) + ticket := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{ + TaskID: "parent", Depth: 1, MaxDepth: 1, + }, func(ctx context.Context, lease *ChildRunLease) { + result <- lease.Continue(ctx, ChildRunConstraints{ + TaskID: "wrong-lineage", ParentTaskID: "someone-else", Depth: 2, MaxDepth: 2, + }, func(context.Context, *ChildRunLease) {}) + result <- lease.Continue(ctx, ChildRunConstraints{ + TaskID: "too-deep", ParentTaskID: "parent", Depth: 2, MaxDepth: 1, + }, func(context.Context, *ChildRunLease) {}) + result <- lease.Continue(ctx, ChildRunConstraints{ + TaskID: "zero-depth", ParentTaskID: "parent", Depth: 0, + }, func(context.Context, *ChildRunLease) {}) + result <- lease.Continue(ctx, ChildRunConstraints{ + TenantID: uuid.New(), TaskID: "other-tenant", ParentTaskID: "parent", Depth: 2, + }, func(context.Context, *ChildRunLease) {}) + }) + waitChildRunSignal(t, ticket.Done(), "lineage validation callback completion") + + if err := waitChildRunValue(t, result, "invalid lineage result"); !errors.Is(err, ErrChildRunInvalidContinuation) { + t.Fatalf("invalid lineage error = %v, want ErrChildRunInvalidContinuation", err) + } + if err := waitChildRunValue(t, result, "depth result"); !errors.Is(err, ErrChildRunContinuationDepth) { + t.Fatalf("depth error = %v, want ErrChildRunContinuationDepth", err) + } + if err := waitChildRunValue(t, result, "zero-depth result"); !errors.Is(err, ErrChildRunInvalidContinuation) { + t.Fatalf("zero-depth error = %v, want ErrChildRunInvalidContinuation", err) + } + if err := waitChildRunValue(t, result, "tenant transition result"); !errors.Is(err, ErrChildRunInvalidContinuation) { + t.Fatalf("tenant transition error = %v, want ErrChildRunInvalidContinuation", err) + } +} + +func TestChildRunAdmissionContinuationPanicResumesParent(t *testing.T) { + controller := NewChildRunAdmission(1, 4) + closeChildRunController(t, controller) + + var resumed atomic.Bool + var continuationErr error + var failedChildLease *ChildRunLease + parent := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{ + TaskID: "parent", Depth: 1, MaxDepth: 3, + }, func(ctx context.Context, lease *ChildRunLease) { + continuationErr = lease.Continue(ctx, ChildRunConstraints{ + TaskID: "child", ParentTaskID: "parent", Depth: 2, MaxDepth: 3, + }, func(_ context.Context, childLease *ChildRunLease) { + failedChildLease = childLease + panic("nested boom") + }) + resumed.Store(true) + }) + waitChildRunSignal(t, parent.Done(), "parent resume after nested panic") + if !resumed.Load() { + t.Fatal("parent did not resume after nested callback panic") + } + if continuationErr == nil { + t.Fatal("nested callback panic did not return an error") + } + if failedChildLease == nil || failedChildLease.frame.state != ChildRunFailed { + t.Fatalf("panicking continuation state = %v, want %q", failedChildLease, ChildRunFailed) + } + if got := parent.State(); got != ChildRunCompleted { + t.Fatalf("parent state after handled nested panic = %q, want %q", got, ChildRunCompleted) + } + if stats := controller.Stats(); stats.Active != 0 || stats.Pending != 0 { + t.Fatalf("stats after nested panic = %+v, want zero active/pending", stats) + } +} + +func TestChildRunAdmissionQueuedContinuationContextCancellationResumesParent(t *testing.T) { + controller := NewChildRunAdmission(2, 4) + closeChildRunController(t, controller) + + tenant := uuid.New() + rootA := uuid.New() + rootB := uuid.New() + rootBRelease := make(chan struct{}) + rootBOwner := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{ + TenantID: tenant, RootAgentID: rootB, RootLimit: 1, TaskID: "root-b-owner", + }, func(context.Context, *ChildRunLease) { + <-rootBRelease + }) + waitChildRunSignal(t, rootBOwner.Started(), "root B owner start") + + childCtx, cancelChild := context.WithCancel(context.Background()) + var childRan atomic.Bool + var continuationErr error + var parentResumed atomic.Bool + parent := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{ + TenantID: tenant, RootAgentID: rootA, RootLimit: 1, + TaskID: "parent-a", Depth: 1, MaxDepth: 3, + }, func(_ context.Context, lease *ChildRunLease) { + continuationErr = lease.Continue(childCtx, ChildRunConstraints{ + TenantID: tenant, RootAgentID: rootB, RootLimit: 1, + TaskID: "child-b", ParentTaskID: "parent-a", ParentFanout: 1, + Depth: 2, MaxDepth: 3, + }, func(context.Context, *ChildRunLease) { + childRan.Store(true) + }) + parentResumed.Store(true) + }) + waitChildRunSignal(t, parent.Started(), "parent A start") + waitForChildRunCondition(t, "queued continuation state", func() bool { + return parent.State() == ChildRunWaitingChild && controller.Stats().Pending == 1 + }) + cancelChild() + waitChildRunSignal(t, parent.Done(), "parent resume after child context cancellation") + if childRan.Load() { + t.Fatal("cancelled queued continuation callback ran") + } + if !parentResumed.Load() { + t.Fatal("parent did not resume after queued child context cancellation") + } + if !errors.Is(continuationErr, context.Canceled) { + t.Fatalf("continuation error = %v, want context.Canceled", continuationErr) + } + close(rootBRelease) +} + +func TestChildRunAdmissionSerializesConcurrentSynchronousContinuations(t *testing.T) { + controller := NewChildRunAdmission(1, 4) + closeChildRunController(t, controller) + + tenant := uuid.New() + root := uuid.New() + var active atomic.Int32 + var maximum atomic.Int32 + var calls atomic.Int32 + childStarted := make(chan struct{}, 2) + releaseChildren := make(chan struct{}) + + parent := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{ + TenantID: tenant, RootAgentID: root, RootLimit: 1, + TaskID: "parent", Depth: 1, MaxDepth: 3, + }, func(ctx context.Context, lease *ChildRunLease) { + var group sync.WaitGroup + for index := range 2 { + index := index + group.Go(func() { + err := lease.Continue(ctx, ChildRunConstraints{ + TenantID: tenant, RootAgentID: root, RootLimit: 1, + TaskID: "child-" + string(rune('a'+index)), ParentTaskID: "parent", + ParentFanout: 1, Depth: 2, MaxDepth: 3, + }, func(context.Context, *ChildRunLease) { + current := active.Add(1) + for { + observed := maximum.Load() + if current <= observed || maximum.CompareAndSwap(observed, current) { + break + } + } + calls.Add(1) + childStarted <- struct{}{} + <-releaseChildren + active.Add(-1) + }) + if err != nil { + t.Errorf("Continue %d: %v", index, err) + } + }) + } + waitChildRunSignal(t, childStarted, "first serialized child start") + assertChildRunNotSignalled(t, childStarted, "second synchronous child overlapped first") + close(releaseChildren) + group.Wait() + }) + waitChildRunSignal(t, parent.Done(), "serialized continuation parent completion") + if got := calls.Load(); got != 2 { + t.Fatalf("child callback count = %d, want 2", got) + } + if got := maximum.Load(); got != 1 { + t.Fatalf("maximum concurrent synchronous children = %d, want 1", got) + } +} + +func TestChildRunAdmissionConstraintChangingContinuationReleasesParentCapacity(t *testing.T) { + controller := NewChildRunAdmission(2, 8) + closeChildRunController(t, controller) + + tenant := uuid.New() + rootA := uuid.New() + rootB := uuid.New() + rootBRelease := make(chan struct{}) + rootBOwner := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{ + TenantID: tenant, RootAgentID: rootB, RootLimit: 1, TaskID: "root-b-owner", + }, func(context.Context, *ChildRunLease) { + <-rootBRelease + }) + waitChildRunSignal(t, rootBOwner.Started(), "root B owner start") + + parentEntered := make(chan struct{}) + childBStarted := make(chan struct{}) + parentResumed := make(chan struct{}) + parent := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{ + TenantID: tenant, RootAgentID: rootA, RootLimit: 1, + TaskID: "parent-a", Depth: 1, MaxDepth: 3, + }, func(ctx context.Context, lease *ChildRunLease) { + close(parentEntered) + err := lease.Continue(ctx, ChildRunConstraints{ + TenantID: tenant, RootAgentID: rootB, RootLimit: 1, + TaskID: "child-b", ParentTaskID: "parent-a", ParentFanout: 1, + Depth: 2, MaxDepth: 3, + }, func(context.Context, *ChildRunLease) { + close(childBStarted) + }) + if err != nil { + t.Errorf("Continue to root B: %v", err) + return + } + close(parentResumed) + }) + waitChildRunSignal(t, parentEntered, "root A parent start") + + unrelated := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{ + TenantID: tenant, RootAgentID: uuid.New(), RootLimit: 1, TaskID: "unrelated", + }, func(context.Context, *ChildRunLease) {}) + waitChildRunSignal(t, unrelated.Done(), "unrelated work while parent is suspended") + assertChildRunNotSignalled(t, childBStarted, "root B child bypassed saturated root") + + close(rootBRelease) + waitChildRunSignal(t, childBStarted, "root B child start") + waitChildRunSignal(t, parentResumed, "root A parent resume") + waitChildRunSignal(t, parent.Done(), "root A parent completion") +} + +func TestChildRunAdmissionCancellationBeforeGrantRemovesWaiter(t *testing.T) { + controller := NewChildRunAdmission(1, 2) + closeChildRunController(t, controller) + + block := make(chan struct{}) + active := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(context.Context, *ChildRunLease) { + <-block + }) + waitChildRunSignal(t, active.Started(), "active run start") + + var ran atomic.Bool + ctx, cancel := context.WithCancel(context.Background()) + queued, err := controller.Enqueue(ctx, ChildRunConstraints{}, func(context.Context, *ChildRunLease) { + ran.Store(true) + }) + if err != nil { + t.Fatalf("Enqueue queued: %v", err) + } + if err := queued.Activate(); err != nil { + t.Fatalf("Activate queued: %v", err) + } + cancel() + waitChildRunSignal(t, queued.Done(), "queued cancellation") + if got := queued.State(); got != ChildRunCancelled { + t.Fatalf("queued state = %q, want %q", got, ChildRunCancelled) + } + if ran.Load() { + t.Fatal("cancelled queued callback ran") + } + if stats := controller.Stats(); stats.Pending != 0 { + t.Fatalf("pending after cancellation = %d, want 0", stats.Pending) + } + close(block) +} + +func TestChildRunAdmissionGrantWinningCancellationRetainsLeaseUntilReturn(t *testing.T) { + controller := NewChildRunAdmission(1, 2) + closeChildRunController(t, controller) + + callbackStarted := make(chan struct{}) + callbackRelease := make(chan struct{}) + var contextCancelled atomic.Bool + first := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(ctx context.Context, lease *ChildRunLease) { + close(callbackStarted) + <-ctx.Done() + contextCancelled.Store(true) + <-callbackRelease + }) + waitChildRunSignal(t, callbackStarted, "cancellation-resistant callback start") + + second := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(context.Context, *ChildRunLease) {}) + if !first.Cancel() { + t.Fatal("Cancel running ticket returned false") + } + if got := first.State(); got != ChildRunCancelled { + t.Fatalf("running cancellation state = %q, want %q", got, ChildRunCancelled) + } + waitForChildRunCondition(t, "callback context cancellation", contextCancelled.Load) + if stats := controller.Stats(); stats.Active != 1 { + t.Fatalf("active after early Release calls = %d, want 1", stats.Active) + } + assertChildRunNotSignalled(t, second.Started(), "second run started before cancelled callback returned") + + close(callbackRelease) + waitChildRunSignal(t, first.Done(), "cancelled callback return") + waitChildRunSignal(t, second.Done(), "second callback completion") +} + +func TestChildRunAdmissionLeaseReleaseReturnsCapacityBeforeCallbackReturn(t *testing.T) { + controller := NewChildRunAdmission(1, 2) + closeChildRunController(t, controller) + + callbackStarted := make(chan struct{}) + leaseReleased := make(chan struct{}) + callbackRelease := make(chan struct{}) + first := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(_ context.Context, lease *ChildRunLease) { + close(callbackStarted) + lease.Release() + lease.Release() + close(leaseReleased) + <-callbackRelease + }) + waitChildRunSignal(t, callbackStarted, "first callback start") + waitChildRunSignal(t, leaseReleased, "lease release") + if stats := controller.Stats(); stats.Active != 0 { + t.Fatalf("active after Release calls = %d, want 0", stats.Active) + } + + second := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(context.Context, *ChildRunLease) {}) + waitChildRunSignal(t, second.Done(), "second run after explicit release") + if stats := controller.Stats(); stats.Live != 1 { + t.Fatalf("live tickets while first callback remains open = %d, want 1", stats.Live) + } + assertChildRunNotSignalled(t, first.Done(), "ticket completed before callback return") + + close(callbackRelease) + waitChildRunSignal(t, first.Done(), "first callback return") +} + +func TestChildRunAdmissionCloseCancelsReleasedLiveCallbackAndDrains(t *testing.T) { + controller := NewChildRunAdmission(1, 2) + + callbackStarted := make(chan struct{}) + leaseReleased := make(chan struct{}) + callbackCancelled := make(chan struct{}) + ticket := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(ctx context.Context, lease *ChildRunLease) { + close(callbackStarted) + lease.Release() + close(leaseReleased) + <-ctx.Done() + close(callbackCancelled) + }) + waitChildRunSignal(t, callbackStarted, "callback start") + waitChildRunSignal(t, leaseReleased, "lease release") + + if stats := controller.Stats(); stats.Active != 0 || stats.Live != 1 { + t.Fatalf("stats after lease release = %+v, want active=0 live=1", stats) + } + + closeCtx, cancelClose := context.WithTimeout(context.Background(), time.Second) + defer cancelClose() + if err := controller.Close(closeCtx); err != nil { + t.Fatalf("Close: %v", err) + } + + waitChildRunSignal(t, callbackCancelled, "callback cancellation") + waitChildRunSignal(t, ticket.Done(), "ticket completion") + if got := ticket.State(); got != ChildRunCancelled { + t.Fatalf("ticket state = %q, want %q", got, ChildRunCancelled) + } + if err := ticket.Err(); !errors.Is(err, ErrChildRunClosed) { + t.Fatalf("ticket error = %v, want ErrChildRunClosed", err) + } + if stats := controller.Stats(); !stats.Closed || stats.Active != 0 || stats.Pending != 0 || stats.Live != 0 { + t.Fatalf("final stats = %+v, want closed and drained", stats) + } +} + +func TestChildRunAdmissionCancellationDuringTransferredChildReleasesExactlyOnce(t *testing.T) { + controller := NewChildRunAdmission(1, 4) + closeChildRunController(t, controller) + + tenant := uuid.New() + root := uuid.New() + childStarted := make(chan struct{}) + childRelease := make(chan struct{}) + parent := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{ + TenantID: tenant, RootAgentID: root, RootLimit: 1, + TaskID: "parent", Depth: 1, MaxDepth: 3, + }, func(ctx context.Context, lease *ChildRunLease) { + _ = lease.Continue(ctx, ChildRunConstraints{ + TenantID: tenant, RootAgentID: root, RootLimit: 1, + TaskID: "child", ParentTaskID: "parent", ParentFanout: 1, + Depth: 2, MaxDepth: 3, + }, func(context.Context, *ChildRunLease) { + close(childStarted) + <-childRelease + }) + }) + waitChildRunSignal(t, childStarted, "transferred child start") + if !parent.Cancel() { + t.Fatal("Cancel parent during child returned false") + } + if stats := controller.Stats(); stats.Active != 1 { + t.Fatalf("active while cancellation-resistant child runs = %d, want 1", stats.Active) + } + close(childRelease) + waitChildRunSignal(t, parent.Done(), "cancelled continuation chain completion") + if stats := controller.Stats(); stats.Active != 0 || stats.Pending != 0 { + t.Fatalf("stats after cancelled continuation = %+v, want zero active/pending", stats) + } + controller.mu.Lock() + defer controller.mu.Unlock() + if len(controller.rootActive) != 0 || len(controller.parentActive) != 0 { + t.Fatalf("constraint counters leaked: root=%v parent=%v", controller.rootActive, controller.parentActive) + } +} + +func TestChildRunAdmissionPanicReleasesCapacityAndMarksFailed(t *testing.T) { + controller := NewChildRunAdmission(1, 2) + closeChildRunController(t, controller) + + failed := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(context.Context, *ChildRunLease) { + panic("boom") + }) + next := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(context.Context, *ChildRunLease) {}) + waitChildRunSignal(t, failed.Done(), "panicking callback completion") + if got := failed.State(); got != ChildRunFailed { + t.Fatalf("panic state = %q, want %q", got, ChildRunFailed) + } + if failed.Err() == nil { + t.Fatal("panic did not record an error") + } + waitChildRunSignal(t, next.Done(), "post-panic callback completion") +} + +func TestChildRunAdmissionCloseIsIdempotentDrainsAndStopsDispatcher(t *testing.T) { + controller := NewChildRunAdmission(1, 2) + callbackStarted := make(chan struct{}) + callbackRelease := make(chan struct{}) + active := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(context.Context, *ChildRunLease) { + close(callbackStarted) + <-callbackRelease + }) + waitChildRunSignal(t, callbackStarted, "active callback start") + + var queuedRan atomic.Bool + queued := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(context.Context, *ChildRunLease) { + queuedRan.Store(true) + }) + closeCtx, cancelClose := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancelClose() + if err := controller.Close(closeCtx); !errors.Is(err, ErrChildRunDrainTimeout) { + t.Fatalf("first Close error = %v, want ErrChildRunDrainTimeout", err) + } + waitChildRunSignal(t, queued.Done(), "queued close cancellation") + if queuedRan.Load() { + t.Fatal("queued callback ran during close") + } + if stats := controller.Stats(); stats.Active != 1 || !stats.Closed { + t.Fatalf("stats after timed-out close = %+v", stats) + } + if _, err := controller.Enqueue(context.Background(), ChildRunConstraints{}, func(context.Context, *ChildRunLease) {}); !errors.Is(err, ErrChildRunClosed) { + t.Fatalf("Enqueue after Close error = %v, want ErrChildRunClosed", err) + } + + close(callbackRelease) + waitChildRunSignal(t, active.Done(), "active callback drain") + if err := controller.Close(context.Background()); err != nil { + t.Fatalf("second Close: %v", err) + } + waitChildRunSignal(t, controller.dispatcherDone, "dispatcher stop") + if err := controller.Close(context.Background()); err != nil { + t.Fatalf("third Close: %v", err) + } +} + +func TestChildRunAdmissionConcurrentCloseCallsShareOneDrain(t *testing.T) { + controller := NewChildRunAdmission(1, 1) + callbackStarted := make(chan struct{}) + callbackRelease := make(chan struct{}) + ticket := enqueueAndActivateChildRun(t, controller, ChildRunConstraints{}, func(context.Context, *ChildRunLease) { + close(callbackStarted) + <-callbackRelease + }) + waitChildRunSignal(t, callbackStarted, "callback start before concurrent close") + + closeResults := make(chan error, 2) + startClose := make(chan struct{}) + for range 2 { + go func() { + <-startClose + closeResults <- controller.Close(context.Background()) + }() + } + close(startClose) + waitForChildRunCondition(t, "controller closed by concurrent caller", func() bool { + return controller.Stats().Closed + }) + close(callbackRelease) + waitChildRunSignal(t, ticket.Done(), "callback drain under concurrent close") + for index := range 2 { + if err := waitChildRunValue(t, closeResults, "concurrent Close result"); err != nil { + t.Fatalf("concurrent Close %d: %v", index, err) + } + } + waitChildRunSignal(t, controller.dispatcherDone, "dispatcher stop after concurrent close") +} + +func enqueueAndActivateChildRun( + t *testing.T, + controller *ChildRunAdmission, + constraints ChildRunConstraints, + run func(context.Context, *ChildRunLease), +) *ChildRunTicket { + t.Helper() + ticket, err := controller.Enqueue(context.Background(), constraints, run) + if err != nil { + t.Fatalf("Enqueue: %v", err) + } + if err := ticket.Activate(); err != nil { + t.Fatalf("Activate: %v", err) + } + return ticket +} + +func closeChildRunController(t *testing.T, controller *ChildRunAdmission) { + t.Helper() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), childRunTestTimeout) + defer cancel() + if err := controller.Close(ctx); err != nil { + t.Errorf("Close: %v", err) + } + }) +} + +func waitChildRunSignal(t *testing.T, signal <-chan struct{}, description string) { + t.Helper() + select { + case <-signal: + case <-time.After(childRunTestTimeout): + t.Fatalf("timed out waiting for %s", description) + } +} + +func assertChildRunNotSignalled(t *testing.T, signal <-chan struct{}, description string) { + t.Helper() + select { + case <-signal: + t.Fatal(description) + case <-time.After(20 * time.Millisecond): + } +} + +func waitChildRunValue[T any](t *testing.T, values <-chan T, description string) T { + t.Helper() + select { + case value := <-values: + return value + case <-time.After(childRunTestTimeout): + t.Fatalf("timed out waiting for %s", description) + var zero T + return zero + } +} + +func waitForChildRunCondition(t *testing.T, description string, condition func() bool) { + t.Helper() + deadline := time.Now().Add(childRunTestTimeout) + for !condition() { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %s", description) + } + time.Sleep(time.Millisecond) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index ce808be9..3f459292 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -606,10 +606,10 @@ func (cc CronConfig) ToRetryConfig() cron.RetryConfig { return cfg } -// SubagentsConfig configures the subagent system (matching TS agents.defaults.subagents). +// SubagentsConfig configures the GoClaw subagent system. // All fields optional — zero values mean "use default". type SubagentsConfig struct { - MaxConcurrent int `json:"maxConcurrent,omitempty"` // default 8 (TS: DEFAULT_SUBAGENT_MAX_CONCURRENT) + MaxConcurrent int `json:"maxConcurrent,omitempty"` // executing descendants per root agent; default 20 MaxSpawnDepth int `json:"maxSpawnDepth,omitempty"` // default 1, range 1-5 MaxChildrenPerAgent int `json:"maxChildrenPerAgent,omitempty"` // default 5, range 1-20 ArchiveAfterMinutes int `json:"archiveAfterMinutes,omitempty"` // default 60 diff --git a/internal/edition/edition.go b/internal/edition/edition.go index fa7869fe..77205df4 100644 --- a/internal/edition/edition.go +++ b/internal/edition/edition.go @@ -16,10 +16,10 @@ type Edition struct { MaxSubagentDepth int `json:"max_subagent_depth"` // 0 = use config default KGEnabled bool `json:"kg_enabled"` RBACEnabled bool `json:"rbac_enabled"` - TeamFullMode bool `json:"team_full_mode"` // false = lite task actions only - VectorSearch bool `json:"vector_search"` // false = FTS5 only - SupportsPipNpm bool `json:"supports_pip_npm"` // false for Lite desktop - SupportsApk bool `json:"supports_apk"` // false for Lite desktop (no apk on macOS/Windows) + TeamFullMode bool `json:"team_full_mode"` // false = lite task actions only + VectorSearch bool `json:"vector_search"` // false = FTS5 only + SupportsPipNpm bool `json:"supports_pip_npm"` // false for Lite desktop + SupportsApk bool `json:"supports_apk"` // false for Lite desktop (no apk on macOS/Windows) } // --- Presets --- @@ -93,3 +93,13 @@ func (e Edition) ChannelLimit(channelType string) int { func (e Edition) AllowsChannels() bool { return e.Name == "standard" } + +// ChildRunLimit returns the process-wide cap for executing self-spawn and +// Agent Link child callbacks. It is separate from the per-agent +// maxConcurrent setting and is intentionally not serialized. +func (e Edition) ChildRunLimit() int { + if e.Name == Lite.Name { + return 2 + } + return 32 +} diff --git a/internal/edition/edition_test.go b/internal/edition/edition_test.go index fe69c5c9..d9695ca7 100644 --- a/internal/edition/edition_test.go +++ b/internal/edition/edition_test.go @@ -295,6 +295,15 @@ func TestLiteEditionFeatures(t *testing.T) { } } +func TestEditionChildRunLimit(t *testing.T) { + if got := Standard.ChildRunLimit(); got != 32 { + t.Fatalf("Standard.ChildRunLimit() = %d, want 32", got) + } + if got := Lite.ChildRunLimit(); got != 2 { + t.Fatalf("Lite.ChildRunLimit() = %d, want 2", got) + } +} + // TestLiteEditionChannelLimits verifies Lite channel constraints. func TestLiteEditionChannelLimits(t *testing.T) { e := Lite diff --git a/internal/gateway/bridge_context_test.go b/internal/gateway/bridge_context_test.go index 04e0ab7a..987dc57b 100644 --- a/internal/gateway/bridge_context_test.go +++ b/internal/gateway/bridge_context_test.go @@ -88,3 +88,85 @@ func TestBridgeContextMiddleware_NoStore_NoAgentKey(t *testing.T) { t.Errorf("ToolAgentKeyFromCtx = %q, want empty when no agent store is wired", gotKey) } } + +func TestBridgeContextMiddleware_InjectsSignedDelegationArtifactContext(t *testing.T) { + const ( + gatewayToken = "test-gateway-token" + delegationID = "delegation-123" + inputsRoot = "/runtime/delegations/delegation-123/inputs" + ) + agentID := uuid.New() + var gotDelegationID, gotInputs string + var artifactRun bool + next := http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + gotDelegationID = tools.DelegationIDFromCtx(r.Context()) + gotInputs = tools.DelegationArtifactInputsFromCtx(r.Context()) + artifactRun = tools.IsDelegationArtifactRun(r.Context()) + }) + mw := bridgeContextMiddleware(gatewayToken, nil, next) + + sig := providers.SignBridgeContext( + gatewayToken, + agentID.String(), + "", + "", + "", + "", + "/runtime/delegations/delegation-123/outputs", + "", + "", + "", + delegationID, + inputsRoot, + ) + req := httptest.NewRequest(http.MethodPost, "/mcp/bridge", nil) + req.Header.Set("X-Agent-ID", agentID.String()) + req.Header.Set("X-Workspace", "/runtime/delegations/delegation-123/outputs") + req.Header.Set("X-Delegation-ID", delegationID) + req.Header.Set("X-Delegation-Inputs", inputsRoot) + req.Header.Set("X-Bridge-Sig", sig) + + rec := httptest.NewRecorder() + mw.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK || !artifactRun { + t.Fatalf("status=%d artifactRun=%v body=%s", rec.Code, artifactRun, rec.Body.String()) + } + if gotDelegationID != delegationID || gotInputs != inputsRoot { + t.Fatalf("delegation context = (%q, %q)", gotDelegationID, gotInputs) + } +} + +func TestBridgeContextMiddleware_RejectsLegacySignatureWithDelegationHeaders(t *testing.T) { + const gatewayToken = "test-gateway-token" + agentID := uuid.New() + legacySig := providers.SignBridgeContext( + gatewayToken, + agentID.String(), + "", + "", + "", + "", + "/runtime/outputs", + "", + "", + "", + ) + req := httptest.NewRequest(http.MethodPost, "/mcp/bridge", nil) + req.Header.Set("X-Agent-ID", agentID.String()) + req.Header.Set("X-Workspace", "/runtime/outputs") + req.Header.Set("X-Delegation-ID", "delegation-123") + req.Header.Set("X-Delegation-Inputs", "/runtime/inputs") + req.Header.Set("X-Bridge-Sig", legacySig) + + var called bool + mw := bridgeContextMiddleware(gatewayToken, nil, http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + called = true + })) + rec := httptest.NewRecorder() + mw.ServeHTTP(rec, req) + + if rec.Code != http.StatusForbidden || called { + t.Fatalf("status=%d called=%v, want forbidden", rec.Code, called) + } +} diff --git a/internal/gateway/server.go b/internal/gateway/server.go index 9c1499bc..9da83bb9 100644 --- a/internal/gateway/server.go +++ b/internal/gateway/server.go @@ -373,6 +373,8 @@ func bridgeContextMiddleware(gatewayToken string, agentStore store.AgentStore, n workspace := r.Header.Get("X-Workspace") localKey := r.Header.Get("X-Local-Key") sessionKey := r.Header.Get("X-Session-Key") + delegationID := r.Header.Get("X-Delegation-ID") + delegationInputs := r.Header.Get("X-Delegation-Inputs") if agentIDStr != "" || userID != "" { // Reject context headers when no gateway token — prevents unauthenticated impersonation. @@ -386,7 +388,44 @@ func bridgeContextMiddleware(gatewayToken string, agentStore store.AgentStore, n // Verify HMAC signature over all context fields. tenantIDStr := r.Header.Get("X-Tenant-ID") sig := r.Header.Get("X-Bridge-Sig") - ok, tenantVerified := providers.VerifyBridgeContext(gatewayToken, agentIDStr, userID, channel, chatID, peerKind, workspace, tenantIDStr, sig, localKey, sessionKey) + var ok, tenantVerified bool + hasDelegationContext := delegationID != "" || delegationInputs != "" + if hasDelegationContext { + if delegationID == "" || delegationInputs == "" { + http.Error(w, `{"error":"incomplete delegation bridge context"}`, http.StatusForbidden) + return + } + expected := providers.SignBridgeContext( + gatewayToken, + agentIDStr, + userID, + channel, + chatID, + peerKind, + workspace, + tenantIDStr, + localKey, + sessionKey, + delegationID, + delegationInputs, + ) + ok = subtle.ConstantTimeCompare([]byte(sig), []byte(expected)) == 1 + tenantVerified = ok + } else { + ok, tenantVerified = providers.VerifyBridgeContext( + gatewayToken, + agentIDStr, + userID, + channel, + chatID, + peerKind, + workspace, + tenantIDStr, + sig, + localKey, + sessionKey, + ) + } if !ok { slog.Warn("security.mcp_bridge: invalid bridge context signature", "agent_id", agentIDStr, "user_id", userID) @@ -448,6 +487,10 @@ func bridgeContextMiddleware(gatewayToken string, agentStore store.AgentStore, n if workspace != "" && (agentIDStr != "" || userID != "") { ctx = tools.WithToolWorkspace(ctx, workspace) } + if delegationID != "" && delegationInputs != "" && (agentIDStr != "" || userID != "") { + ctx = tools.WithDelegationID(ctx, delegationID) + ctx = tools.WithDelegationArtifactInputs(ctx, delegationInputs) + } // Routing context (localKey, sessionKey) is injected unconditionally like channel/chatID. // These are used for message routing (forum topics), not security-sensitive operations. // Without valid agent context, tool execution will fail anyway. @@ -514,9 +557,9 @@ func (s *Server) Start(ctx context.Context) error { s.httpServer = &http.Server{ Addr: addr, Handler: handler, - ReadTimeout: 3600 * time.Second, // 1h: allow large uploads, long-running reads - WriteTimeout: 3600 * time.Second, // 1h: allow streaming responses, slow clients - IdleTimeout: 30 * time.Second, // 30s: close idle connections + ReadTimeout: 3600 * time.Second, // 1h: allow large uploads, long-running reads + WriteTimeout: 3600 * time.Second, // 1h: allow streaming responses, slow clients + IdleTimeout: 30 * time.Second, // 30s: close idle connections } slog.Info("gateway starting", "addr", addr) diff --git a/internal/hooks/tracing.go b/internal/hooks/tracing.go index 5bd18e0d..f164accb 100644 --- a/internal/hooks/tracing.go +++ b/internal/hooks/tracing.go @@ -107,5 +107,5 @@ func EmitHookSpan( if collector == nil { return // tracing disabled — no collector attached } - collector.EmitSpan(span) + collector.EmitSpan(tracing.RedactSpan(ctx, span)) } diff --git a/internal/mcp/bridge_server.go b/internal/mcp/bridge_server.go index a5a043e1..621be107 100644 --- a/internal/mcp/bridge_server.go +++ b/internal/mcp/bridge_server.go @@ -205,6 +205,10 @@ func makeToolHandler(reg *tools.Registry, toolName string, msgBus *bus.MessageBu return mcpgo.NewToolResultError(result.ForLLM), nil } + // A delegated artifact is not outbound media until DelegateTool validates + // and atomically publishes its manifest back into the caller workspace. + tools.ApplyDelegationArtifactResultPolicy(ctx, result) + // Forward media files to the outbound bus so they reach the user as attachments. // This is necessary because Claude CLI processes tool results internally — // GoClaw's agent loop never sees result.Media from bridge tool calls. @@ -216,7 +220,7 @@ func makeToolHandler(reg *tools.Registry, toolName string, msgBus *bus.MessageBu // forwardMediaToOutbound publishes media files from a tool result to the outbound bus. func forwardMediaToOutbound(ctx context.Context, msgBus *bus.MessageBus, toolName string, result *tools.Result) { - if msgBus == nil || len(result.Media) == 0 { + if msgBus == nil || tools.IsDelegationArtifactRun(ctx) || len(result.Media) == 0 { return } channel := tools.ToolChannelFromCtx(ctx) diff --git a/internal/mcp/bridge_server_test.go b/internal/mcp/bridge_server_test.go index fd98d4cb..a466fc01 100644 --- a/internal/mcp/bridge_server_test.go +++ b/internal/mcp/bridge_server_test.go @@ -3,9 +3,11 @@ package mcp import ( "context" "testing" + "time" mcpgo "github.com/mark3labs/mcp-go/mcp" + "github.com/nextlevelbuilder/goclaw/internal/bus" "github.com/nextlevelbuilder/goclaw/internal/config" "github.com/nextlevelbuilder/goclaw/internal/tools" ) @@ -141,3 +143,21 @@ func toolNames(ts []mcpgo.Tool) []string { } return out } + +func TestForwardMediaToOutboundSkipsDelegationArtifactRun(t *testing.T) { + msgBus := bus.New() + ctx := tools.WithToolChannel(context.Background(), "telegram") + ctx = tools.WithToolChatID(ctx, "chat-id") + ctx = tools.WithDelegationID(ctx, "delegation-id") + ctx = tools.WithDelegationArtifactInputs(ctx, "/runtime/inputs") + + forwardMediaToOutbound(ctx, msgBus, "create_image", &tools.Result{ + Media: []bus.MediaFile{{Path: "/runtime/outputs/generated.png", MimeType: "image/png"}}, + }) + + waitCtx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + if msg, ok := msgBus.SubscribeOutbound(waitCtx); ok { + t.Fatalf("unpublished delegation media forwarded: %#v", msg) + } +} diff --git a/internal/mcp/crud_agent_links.go b/internal/mcp/crud_agent_links.go index f00c8dcc..b201f345 100644 --- a/internal/mcp/crud_agent_links.go +++ b/internal/mcp/crud_agent_links.go @@ -28,7 +28,7 @@ func registerAgentLinkCRUDTools(srv *mcpserver.MCPServer, links store.AgentLinkS mcpgo.WithString("target_agent", mcpgo.Required(), mcpgo.Description("Target agent UUID.")), mcpgo.WithString("direction", mcpgo.Enum("outbound", "inbound", "bidirectional"), mcpgo.Description("Link direction; defaults to \"outbound\".")), mcpgo.WithString("description", mcpgo.Description("Human-readable description of the link.")), - mcpgo.WithNumber("max_concurrent", mcpgo.Description("Maximum concurrent delegated tasks.")), + mcpgo.WithNumber("max_concurrent", mcpgo.Description("Compatibility metadata reserved for future per-link scheduling; not enforced at runtime.")), ), handleAgentLinksCreate(links)) srv.AddTool(mcpgo.NewTool("goclaw_agent_links_update", @@ -36,7 +36,7 @@ func registerAgentLinkCRUDTools(srv *mcpserver.MCPServer, links store.AgentLinkS mcpgo.WithString("link_id", mcpgo.Required(), mcpgo.Description("Link UUID.")), mcpgo.WithString("direction", mcpgo.Enum("outbound", "inbound", "bidirectional"), mcpgo.Description("New direction.")), mcpgo.WithString("description", mcpgo.Description("New description.")), - mcpgo.WithNumber("max_concurrent", mcpgo.Description("New max concurrent delegated tasks.")), + mcpgo.WithNumber("max_concurrent", mcpgo.Description("New compatibility metadata value reserved for future per-link scheduling; not enforced at runtime.")), mcpgo.WithString("status", mcpgo.Enum("active", "disabled"), mcpgo.Description("New status.")), ), handleAgentLinksUpdate(links)) diff --git a/internal/mcp/crud_agent_links_contract_test.go b/internal/mcp/crud_agent_links_contract_test.go new file mode 100644 index 00000000..89da2c6e --- /dev/null +++ b/internal/mcp/crud_agent_links_contract_test.go @@ -0,0 +1,38 @@ +package mcp + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +func TestAgentLinkMaxConcurrentSchemaIsCompatibilityMetadata(t *testing.T) { + srv := newTestMCPServer() + var links store.AgentLinkStore + registerAgentLinkCRUDTools(srv, links) + + for _, toolName := range []string{ + "goclaw_agent_links_create", + "goclaw_agent_links_update", + } { + registered := srv.GetTool(toolName) + if registered == nil { + t.Fatalf("%s was not registered", toolName) + } + property, ok := registered.Tool.InputSchema.Properties["max_concurrent"] + if !ok { + t.Fatalf("%s omitted max_concurrent compatibility field", toolName) + } + encoded, err := json.Marshal(property) + if err != nil { + t.Fatalf("marshal %s max_concurrent schema: %v", toolName, err) + } + description := strings.ToLower(string(encoded)) + if !strings.Contains(description, "compatibility metadata") || + !strings.Contains(description, "not enforced") { + t.Fatalf("%s exposes a misleading max_concurrent contract: %s", toolName, encoded) + } + } +} diff --git a/internal/media/store.go b/internal/media/store.go index f9cada68..bdded68f 100644 --- a/internal/media/store.go +++ b/internal/media/store.go @@ -70,6 +70,12 @@ func (s *Store) LoadPath(id string) (string, error) { return matches[0], nil } +// MediaRootPath exposes the managed legacy-media boundary to media reader +// tools. Returned paths are still containment-checked by the caller. +func (s *Store) MediaRootPath() string { + return s.baseDir +} + // DeleteSession removes all media files for a session. func (s *Store) DeleteSession(sessionKey string) error { dir := s.sessionDir(sessionKey) diff --git a/internal/orchestration/child_run_admission.go b/internal/orchestration/child_run_admission.go new file mode 100644 index 00000000..00f2677c --- /dev/null +++ b/internal/orchestration/child_run_admission.go @@ -0,0 +1,33 @@ +package orchestration + +import "github.com/nextlevelbuilder/goclaw/internal/childrun" + +var ( + ErrChildRunBusy = childrun.ErrChildRunBusy + ErrChildRunClosed = childrun.ErrChildRunClosed + ErrChildRunDrainTimeout = childrun.ErrChildRunDrainTimeout + ErrChildRunInvalidContinuation = childrun.ErrChildRunInvalidContinuation + ErrChildRunContinuationDepth = childrun.ErrChildRunContinuationDepth +) + +type ChildRunState = childrun.ChildRunState + +const ( + ChildRunNew = childrun.ChildRunNew + ChildRunQueued = childrun.ChildRunQueued + ChildRunRunning = childrun.ChildRunRunning + ChildRunWaitingChild = childrun.ChildRunWaitingChild + ChildRunCompleted = childrun.ChildRunCompleted + ChildRunFailed = childrun.ChildRunFailed + ChildRunCancelled = childrun.ChildRunCancelled +) + +type ChildRunConstraints = childrun.ChildRunConstraints +type ChildRunStats = childrun.ChildRunStats +type ChildRunAdmission = childrun.ChildRunAdmission +type ChildRunTicket = childrun.ChildRunTicket +type ChildRunLease = childrun.ChildRunLease + +func NewChildRunAdmission(processLimit, pendingLimit int) *ChildRunAdmission { + return childrun.NewChildRunAdmission(processLimit, pendingLimit) +} diff --git a/internal/providers/claude_cli.go b/internal/providers/claude_cli.go index 5f54b2b6..69bdbbdd 100644 --- a/internal/providers/claude_cli.go +++ b/internal/providers/claude_cli.go @@ -32,6 +32,11 @@ const OptPeerKind = "peer_kind" // OptWorkspace passes the agent workspace path so MCP bridge tools can resolve file paths. const OptWorkspace = "workspace" +// OptDelegationID and OptDelegationInputs pass signed artifact-run context to +// MCP bridge tools. They are runtime-only and are not included in model prompts. +const OptDelegationID = "delegation_id" +const OptDelegationInputs = "delegation_inputs" + // OptTenantID passes the tenant UUID string for per-session MCP config. // Required for memory indexing and tenant-scoped queries via bridge tools. const OptTenantID = "tenant_id" diff --git a/internal/providers/claude_cli_mcp.go b/internal/providers/claude_cli_mcp.go index df03d080..2e3e2ebc 100644 --- a/internal/providers/claude_cli_mcp.go +++ b/internal/providers/claude_cli_mcp.go @@ -84,14 +84,16 @@ func mcpConfigBaseDir() string { // BridgeContext holds per-call context for MCP bridge headers. type BridgeContext struct { - AgentID string - UserID string - Channel string - ChatID string - PeerKind string - Workspace string - TenantID string - LocalKey string + AgentID string + UserID string + Channel string + ChatID string + PeerKind string + Workspace string + TenantID string + LocalKey string + DelegationID string + DelegationInputs string } // WriteMCPConfig writes a per-session MCP config file with agent context headers. @@ -99,10 +101,27 @@ type BridgeContext struct { // outside the agent's workDir so tokens are not exposed. // Skips write if content is unchanged. Returns the file path. func (d *MCPConfigData) WriteMCPConfig(ctx context.Context, sessionKey string, bc BridgeContext) string { - return d.writeMCPConfigInternal(ctx, sessionKey, bc.AgentID, bc.UserID, bc.Channel, bc.ChatID, bc.PeerKind, bc.Workspace, bc.TenantID, bc.LocalKey) + return d.writeMCPConfigInternal( + ctx, + sessionKey, + bc.AgentID, + bc.UserID, + bc.Channel, + bc.ChatID, + bc.PeerKind, + bc.Workspace, + bc.TenantID, + bc.LocalKey, + bc.DelegationID, + bc.DelegationInputs, + ) } -func (d *MCPConfigData) writeMCPConfigInternal(ctx context.Context, sessionKey, agentID, userID, channel, chatID, peerKind, workspace, tenantID, localKey string) string { +func (d *MCPConfigData) writeMCPConfigInternal( + ctx context.Context, + sessionKey, agentID, userID, channel, chatID, peerKind, workspace, tenantID, localKey string, + delegationID, delegationInputs string, +) string { if d == nil || (len(d.Servers) == 0 && d.GatewayAddr == "" && d.AgentMCPLookup == nil) { return "" } @@ -158,9 +177,29 @@ func (d *MCPConfigData) writeMCPConfigInternal(ctx context.Context, sessionKey, if sessionKey != "" && !strings.ContainsAny(sessionKey, "\r\n\x00") { headers["X-Session-Key"] = sessionKey } + if delegationID != "" && !strings.ContainsAny(delegationID, "\r\n\x00") { + headers["X-Delegation-ID"] = delegationID + } + if delegationInputs != "" && !strings.ContainsAny(delegationInputs, "\r\n\x00") { + headers["X-Delegation-Inputs"] = delegationInputs + } // HMAC signature over all context fields to prevent header forgery if d.GatewayToken != "" && (agentID != "" || userID != "") { - headers["X-Bridge-Sig"] = SignBridgeContext(d.GatewayToken, agentID, userID, channel, chatID, peerKind, workspace, tenantID, localKey, sessionKey) + extra := []string{localKey, sessionKey} + if delegationID != "" || delegationInputs != "" { + extra = append(extra, delegationID, delegationInputs) + } + headers["X-Bridge-Sig"] = SignBridgeContext( + d.GatewayToken, + agentID, + userID, + channel, + chatID, + peerKind, + workspace, + tenantID, + extra..., + ) } bridgeEntry := map[string]any{ diff --git a/internal/providers/claude_cli_mcp_test.go b/internal/providers/claude_cli_mcp_test.go index 7622557a..098f0257 100644 --- a/internal/providers/claude_cli_mcp_test.go +++ b/internal/providers/claude_cli_mcp_test.go @@ -1,6 +1,9 @@ package providers import ( + "context" + "encoding/json" + "os" "testing" ) @@ -179,3 +182,66 @@ func TestVerifyBridgeContext_ExtraParamOrderMatters(t *testing.T) { t.Error("expected ok=false for swapped extra param order") } } + +func TestBridgeContextFromOptsCarriesDelegationArtifactContext(t *testing.T) { + got := bridgeContextFromOpts(map[string]any{ + OptAgentID: "agent-id", + OptWorkspace: "/runtime/outputs", + OptDelegationID: "delegation-id", + OptDelegationInputs: "/runtime/inputs", + }) + + if got.DelegationID != "delegation-id" || got.DelegationInputs != "/runtime/inputs" { + t.Fatalf("delegation bridge context = %#v", got) + } +} + +func TestWriteMCPConfigSignsDelegationArtifactHeaders(t *testing.T) { + t.Setenv("GOCLAW_DATA_DIR", t.TempDir()) + const ( + token = "gateway-token" + sessionKey = "session-key" + ) + data := &MCPConfigData{GatewayAddr: "127.0.0.1:18790", GatewayToken: token} + bc := BridgeContext{ + AgentID: "agent-id", + Workspace: "/runtime/outputs", + DelegationID: "delegation-id", + DelegationInputs: "/runtime/inputs", + } + path := data.WriteMCPConfig(context.Background(), sessionKey, bc) + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var cfg struct { + MCPServers map[string]struct { + Headers map[string]string `json:"headers"` + } `json:"mcpServers"` + } + if err := json.Unmarshal(raw, &cfg); err != nil { + t.Fatal(err) + } + headers := cfg.MCPServers["goclaw-bridge"].Headers + if headers["X-Delegation-ID"] != bc.DelegationID || + headers["X-Delegation-Inputs"] != bc.DelegationInputs { + t.Fatalf("delegation headers = %#v", headers) + } + wantSig := SignBridgeContext( + token, + bc.AgentID, + "", + "", + "", + "", + bc.Workspace, + "", + "", + sessionKey, + bc.DelegationID, + bc.DelegationInputs, + ) + if headers["X-Bridge-Sig"] != wantSig { + t.Fatal("delegation headers were not covered by the exact bridge signature") + } +} diff --git a/internal/providers/claude_cli_session.go b/internal/providers/claude_cli_session.go index 488ebdf1..2f0d8c85 100644 --- a/internal/providers/claude_cli_session.go +++ b/internal/providers/claude_cli_session.go @@ -285,14 +285,16 @@ func extractStringSliceOpt(opts map[string]any, key string) []string { // bridgeContextFromOpts builds a BridgeContext from the Options map. func bridgeContextFromOpts(opts map[string]any) BridgeContext { return BridgeContext{ - AgentID: extractStringOpt(opts, OptAgentID), - UserID: extractStringOpt(opts, OptUserID), - Channel: extractStringOpt(opts, OptChannel), - ChatID: extractStringOpt(opts, OptChatID), - PeerKind: extractStringOpt(opts, OptPeerKind), - Workspace: extractStringOpt(opts, OptWorkspace), - TenantID: extractStringOpt(opts, OptTenantID), - LocalKey: extractStringOpt(opts, OptLocalKey), + AgentID: extractStringOpt(opts, OptAgentID), + UserID: extractStringOpt(opts, OptUserID), + Channel: extractStringOpt(opts, OptChannel), + ChatID: extractStringOpt(opts, OptChatID), + PeerKind: extractStringOpt(opts, OptPeerKind), + Workspace: extractStringOpt(opts, OptWorkspace), + TenantID: extractStringOpt(opts, OptTenantID), + LocalKey: extractStringOpt(opts, OptLocalKey), + DelegationID: extractStringOpt(opts, OptDelegationID), + DelegationInputs: extractStringOpt(opts, OptDelegationInputs), } } diff --git a/internal/sandbox/docker.go b/internal/sandbox/docker.go index 044b88c0..733f1fb3 100644 --- a/internal/sandbox/docker.go +++ b/internal/sandbox/docker.go @@ -3,8 +3,6 @@ package sandbox import ( "bytes" "context" - "crypto/sha256" - "encoding/hex" "fmt" "log/slog" "maps" @@ -34,12 +32,19 @@ type DockerSandbox struct { workspace string createdAt time.Time lastUsed time.Time + destroyFunc func(context.Context) error mu sync.Mutex // protects lastUsed } // newDockerSandbox creates and starts a Docker container for sandboxed execution. // Matching TS buildSandboxCreateArgs() + createSandboxContainer(). -func newDockerSandbox(ctx context.Context, name string, cfg Config, workspace string) (*DockerSandbox, error) { +func newDockerSandbox( + ctx context.Context, + name string, + cfg Config, + workspace string, + readOnlyMounts ...ReadOnlyMount, +) (*DockerSandbox, error) { args := []string{ "run", "-d", "--name", name, @@ -90,17 +95,18 @@ func newDockerSandbox(ctx context.Context, name string, cfg Config, workspace st args = append(args, "--network", "none") } - // Workspace mount — resolve host path for DooD (Docker-out-of-Docker) setups. - containerWorkdir := cfg.ContainerWorkdir() - if workspace != "" && cfg.WorkspaceAccess != AccessNone { - mountOpt := "rw" - if cfg.WorkspaceAccess == AccessRO { - mountOpt = "ro" - } - hostPath := resolveHostWorkspacePath(ctx, workspace) - args = append(args, "-v", fmt.Sprintf("%s:%s:%s", hostPath, containerWorkdir, mountOpt)) + args, sensitiveMountRoots, err := appendDockerMountArgs( + ctx, + args, + cfg, + workspace, + readOnlyMounts, + resolveHostWorkspacePath, + ) + if err != nil { + return nil, err } - args = append(args, "-w", containerWorkdir) + args = append(args, "-w", cfg.ContainerWorkdir()) // Environment variables for k, v := range cfg.Env { @@ -110,7 +116,12 @@ func newDockerSandbox(ctx context.Context, name string, cfg Config, workspace st // Image + keep-alive command args = append(args, cfg.Image, "sleep", "infinity") - slog.Debug("creating sandbox container", "name", name, "args", args) + slog.Debug( + "creating sandbox container", + "name", name, + "image", cfg.Image, + "additional_read_only_mounts", len(readOnlyMounts), + ) cmd := exec.CommandContext(ctx, "docker", args...) var stdout, stderr bytes.Buffer @@ -118,7 +129,8 @@ func newDockerSandbox(ctx context.Context, name string, cfg Config, workspace st cmd.Stderr = &stderr if err := cmd.Run(); err != nil { - return nil, fmt.Errorf("docker run failed: %w\nstderr: %s", err, stderr.String()) + safeStderr := redactMountRoots(stderr.String(), sensitiveMountRoots) + return nil, fmt.Errorf("docker run failed: %w\nstderr: %s", err, safeStderr) } containerID := strings.TrimSpace(stdout.String()) @@ -214,6 +226,9 @@ func (s *DockerSandbox) Exec(ctx context.Context, command []string, workDir stri // Destroy removes the container. func (s *DockerSandbox) Destroy(ctx context.Context) error { + if s.destroyFunc != nil { + return s.destroyFunc(ctx) + } cmd := exec.CommandContext(ctx, "docker", "rm", "-f", s.containerID) if err := cmd.Run(); err != nil { slog.Warn("failed to remove sandbox container", "id", s.containerID, "error", err) @@ -234,6 +249,8 @@ type DockerManager struct { stopCh chan struct{} // signals pruning goroutine to stop } +var _ Manager = (*DockerManager)(nil) + // NewDockerManager creates a manager for Docker sandboxes. // Automatically starts background pruning if configured. func NewDockerManager(cfg Config) *DockerManager { @@ -248,7 +265,13 @@ func NewDockerManager(cfg Config) *DockerManager { // Get returns an existing sandbox or creates a new one for the given key. // If cfgOverride is non-nil, it is used for new containers instead of the global config. -func (m *DockerManager) Get(ctx context.Context, key string, workspace string, cfgOverride *Config) (Sandbox, error) { +func (m *DockerManager) Get( + ctx context.Context, + key string, + workspace string, + cfgOverride *Config, + opts ...GetOption, +) (Sandbox, error) { cfg := m.config if cfgOverride != nil { cfg = *cfgOverride @@ -256,7 +279,13 @@ func (m *DockerManager) Get(ctx context.Context, key string, workspace string, c if cfg.Mode == ModeOff { return nil, ErrSandboxDisabled } - cacheKey := dockerCacheKey(key, workspace, cfg) + getOpts := ApplyGetOpts(opts) + cfg = applyGetOptsToConfig(cfg, getOpts) + readOnlyMounts, err := validateReadOnlyMounts(cfg, getOpts.ReadOnlyMounts) + if err != nil { + return nil, err + } + cacheKey := dockerCacheKey(key, workspace, cfg, readOnlyMounts...) m.mu.RLock() if sb, ok := m.sandboxes[cacheKey]; ok { @@ -278,7 +307,7 @@ func (m *DockerManager) Get(ctx context.Context, key string, workspace string, c prefix = "goclaw-sbx-" } name := prefix + sanitizeKey(cacheKey) - sb, err := newDockerSandbox(ctx, name, cfg, workspace) + sb, err := newDockerSandbox(ctx, name, cfg, workspace, readOnlyMounts...) if err != nil { return nil, err } @@ -287,35 +316,21 @@ func (m *DockerManager) Get(ctx context.Context, key string, workspace string, c return sb, nil } -func dockerCacheKey(key, workspace string, cfg Config) string { - if workspace == "" { - return key - } - h := sha256.Sum256([]byte(strings.Join([]string{ - workspace, - string(cfg.WorkspaceAccess), - cfg.ContainerWorkdir(), - cfg.Image, - }, "\x00"))) - return "w" + hex.EncodeToString(h[:])[:16] + ":" + key -} - // Release destroys a sandbox by key. func (m *DockerManager) Release(ctx context.Context, key string) error { m.mu.Lock() - sbs := make([]*DockerSandbox, 0, 1) - for cacheKey, sb := range m.sandboxes { - if cacheKey == key || strings.HasSuffix(cacheKey, ":"+key) { - delete(m.sandboxes, cacheKey) - sbs = append(sbs, sb) - } - } - m.mu.Unlock() + defer m.mu.Unlock() var firstErr error - for _, sb := range sbs { - if err := sb.Destroy(ctx); err != nil && firstErr == nil { - firstErr = err + for cacheKey, sb := range m.sandboxes { + if cacheKey == key || strings.HasSuffix(cacheKey, ":"+key) { + if err := sb.Destroy(ctx); err != nil { + if firstErr == nil { + firstErr = err + } + continue + } + delete(m.sandboxes, cacheKey) } } return firstErr diff --git a/internal/sandbox/docker_cache_key_test.go b/internal/sandbox/docker_cache_key_test.go index c993c86c..4ac43027 100644 --- a/internal/sandbox/docker_cache_key_test.go +++ b/internal/sandbox/docker_cache_key_test.go @@ -1,6 +1,9 @@ package sandbox -import "testing" +import ( + "strings" + "testing" +) func TestDockerCacheKeyIncludesWorkspaceAndConfig(t *testing.T) { cfg := DefaultConfig() @@ -28,3 +31,37 @@ func TestDockerCacheKeyPreservesEmptyWorkspaceCompatibility(t *testing.T) { t.Fatalf("dockerCacheKey empty workspace = %q, want original key %q", got, key) } } + +func TestDockerCacheKeyDoesNotExposeMountRoots(t *testing.T) { + cfg := DefaultConfig() + hostRoot := "/sensitive/runtime/delegation/inputs" + got := dockerCacheKey( + "delegation:9db9", + "/sensitive/runtime/delegation/outputs", + cfg, + ReadOnlyMount{ + Name: "inputs", + HostPath: hostRoot, + Destination: "/workspace/inputs", + }, + ) + + if strings.Contains(got, "/sensitive/") || strings.Contains(got, hostRoot) { + t.Fatalf("dockerCacheKey exposed mount root: %q", got) + } +} + +func TestDockerCacheKeyIncludesDelegationIdentity(t *testing.T) { + cfg := DefaultConfig() + mount := ReadOnlyMount{ + Name: "inputs", + HostPath: "/runtime/inputs", + Destination: "/workspace/inputs", + } + + first := dockerCacheKey("delegation:first", "/runtime/outputs", cfg, mount) + second := dockerCacheKey("delegation:second", "/runtime/outputs", cfg, mount) + if first == second { + t.Fatalf("dockerCacheKey reused identity across delegation keys: %q", first) + } +} diff --git a/internal/sandbox/docker_mounts.go b/internal/sandbox/docker_mounts.go new file mode 100644 index 00000000..839eeed1 --- /dev/null +++ b/internal/sandbox/docker_mounts.go @@ -0,0 +1,186 @@ +package sandbox + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path" + "path/filepath" + "slices" + "sort" + "strings" +) + +const redactedMountRoot = "[REDACTED_MOUNT_ROOT]" + +type mountPathResolver func(context.Context, string) string + +func validateReadOnlyMounts(cfg Config, mounts []ReadOnlyMount) ([]ReadOnlyMount, error) { + if len(mounts) == 0 { + return nil, nil + } + if cfg.WorkspaceAccess == AccessNone { + return nil, fmt.Errorf("read-only mounts require sandbox workspace access") + } + + workdir := cfg.ContainerWorkdir() + if !path.IsAbs(workdir) || path.Clean(workdir) != workdir { + return nil, fmt.Errorf("sandbox workdir must be an absolute canonical path") + } + + normalized := append([]ReadOnlyMount(nil), mounts...) + names := make(map[string]struct{}, len(normalized)) + destinations := make(map[string]struct{}, len(normalized)) + + for i := range normalized { + mount := &normalized[i] + if mount.Name == "" || strings.TrimSpace(mount.Name) != mount.Name || strings.ContainsRune(mount.Name, '\x00') { + return nil, fmt.Errorf("read-only mount %d has an invalid name", i) + } + if _, exists := names[mount.Name]; exists { + return nil, fmt.Errorf("read-only mounts must have unique names") + } + names[mount.Name] = struct{}{} + + if !filepath.IsAbs(mount.HostPath) || filepath.Clean(mount.HostPath) != mount.HostPath { + return nil, fmt.Errorf("read-only mount %d host path must be absolute and canonical", i) + } + canonicalHost, err := filepath.EvalSymlinks(mount.HostPath) + if err != nil { + return nil, fmt.Errorf("read-only mount %d host path cannot be resolved", i) + } + if canonicalHost != mount.HostPath { + return nil, fmt.Errorf("read-only mount %d host path must be absolute and canonical", i) + } + info, err := os.Stat(mount.HostPath) + if err != nil { + return nil, fmt.Errorf("read-only mount %d host path cannot be inspected", i) + } + if !info.IsDir() { + return nil, fmt.Errorf("read-only mount %d host path must be a directory", i) + } + + if strings.ContainsRune(mount.Destination, '\x00') || + !path.IsAbs(mount.Destination) || + path.Clean(mount.Destination) != mount.Destination { + return nil, fmt.Errorf("read-only mount %d destination must be absolute and canonical", i) + } + if !containerPathStrictlyWithin(workdir, mount.Destination) { + return nil, fmt.Errorf("read-only mount %d destination must be strictly beneath sandbox workdir", i) + } + if _, exists := destinations[mount.Destination]; exists { + return nil, fmt.Errorf("read-only mounts must have unique destinations") + } + destinations[mount.Destination] = struct{}{} + } + + sort.Slice(normalized, func(i, j int) bool { + if normalized[i].Name != normalized[j].Name { + return normalized[i].Name < normalized[j].Name + } + if normalized[i].Destination != normalized[j].Destination { + return normalized[i].Destination < normalized[j].Destination + } + return normalized[i].HostPath < normalized[j].HostPath + }) + return normalized, nil +} + +func containerPathStrictlyWithin(root, target string) bool { + if target == root { + return false + } + return strings.HasPrefix(target, strings.TrimSuffix(root, "/")+"/") +} + +func appendDockerMountArgs( + ctx context.Context, + args []string, + cfg Config, + workspace string, + mounts []ReadOnlyMount, + resolve mountPathResolver, +) ([]string, []string, error) { + normalized, err := validateReadOnlyMounts(cfg, mounts) + if err != nil { + return nil, nil, err + } + if resolve == nil { + resolve = func(_ context.Context, path string) string { return path } + } + + sensitiveRoots := make([]string, 0, 2+len(normalized)*2) + containerWorkdir := cfg.ContainerWorkdir() + if workspace != "" && cfg.WorkspaceAccess != AccessNone { + mountOpt := "rw" + if cfg.WorkspaceAccess == AccessRO { + mountOpt = "ro" + } + hostPath := resolve(ctx, workspace) + sensitiveRoots = appendSensitiveRoot(sensitiveRoots, workspace) + sensitiveRoots = appendSensitiveRoot(sensitiveRoots, hostPath) + args = append(args, "-v", fmt.Sprintf("%s:%s:%s", hostPath, containerWorkdir, mountOpt)) + } + + for _, mount := range normalized { + hostPath := resolve(ctx, mount.HostPath) + sensitiveRoots = appendSensitiveRoot(sensitiveRoots, mount.HostPath) + sensitiveRoots = appendSensitiveRoot(sensitiveRoots, hostPath) + args = append(args, "-v", fmt.Sprintf("%s:%s:ro", hostPath, mount.Destination)) + } + return args, sensitiveRoots, nil +} + +func appendSensitiveRoot(roots []string, root string) []string { + if root == "" { + return roots + } + if slices.Contains(roots, root) { + return roots + } + return append(roots, root) +} + +func redactMountRoots(message string, roots []string) string { + sorted := append([]string(nil), roots...) + sort.Slice(sorted, func(i, j int) bool { + return len(sorted[i]) > len(sorted[j]) + }) + for _, root := range sorted { + if root != "" { + message = strings.ReplaceAll(message, root, redactedMountRoot) + } + } + return message +} + +func dockerCacheKey(key, workspace string, cfg Config, mounts ...ReadOnlyMount) string { + if workspace == "" && len(mounts) == 0 { + return key + } + + orderedMounts := append([]ReadOnlyMount(nil), mounts...) + sort.Slice(orderedMounts, func(i, j int) bool { + if orderedMounts[i].Name != orderedMounts[j].Name { + return orderedMounts[i].Name < orderedMounts[j].Name + } + if orderedMounts[i].Destination != orderedMounts[j].Destination { + return orderedMounts[i].Destination < orderedMounts[j].Destination + } + return orderedMounts[i].HostPath < orderedMounts[j].HostPath + }) + + parts := []string{ + workspace, + string(cfg.WorkspaceAccess), + cfg.ContainerWorkdir(), + cfg.Image, + } + for _, mount := range orderedMounts { + parts = append(parts, mount.Name, mount.HostPath, mount.Destination, "ro") + } + h := sha256.Sum256([]byte(strings.Join(parts, "\x00"))) + return "w" + hex.EncodeToString(h[:])[:16] + ":" + key +} diff --git a/internal/sandbox/docker_mounts_test.go b/internal/sandbox/docker_mounts_test.go new file mode 100644 index 00000000..c2579d2c --- /dev/null +++ b/internal/sandbox/docker_mounts_test.go @@ -0,0 +1,342 @@ +package sandbox + +import ( + "context" + "errors" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" +) + +func TestValidateReadOnlyMountsCanonicalizesOrder(t *testing.T) { + first := canonicalTempDir(t) + second := canonicalTempDir(t) + cfg := DefaultConfig() + + got, err := validateReadOnlyMounts(cfg, []ReadOnlyMount{ + {Name: "zeta", HostPath: second, Destination: "/workspace/zeta"}, + {Name: "alpha", HostPath: first, Destination: "/workspace/alpha"}, + }) + if err != nil { + t.Fatalf("validateReadOnlyMounts() error = %v", err) + } + if got[0].Name != "alpha" || got[1].Name != "zeta" { + t.Fatalf("validateReadOnlyMounts() order = %#v, want name-sorted mounts", got) + } +} + +func TestValidateReadOnlyMountsRejectsInvalidContractsWithoutLeakingRoots(t *testing.T) { + host := canonicalTempDir(t) + otherHost := canonicalTempDir(t) + file := filepath.Join(host, "file.txt") + if err := os.WriteFile(file, []byte("test"), 0o600); err != nil { + t.Fatalf("write fixture: %v", err) + } + symlink := filepath.Join(otherHost, "linked") + if err := os.Symlink(host, symlink); err != nil { + t.Fatalf("create symlink fixture: %v", err) + } + + cfg := DefaultConfig() + tests := []struct { + name string + cfg Config + mounts []ReadOnlyMount + }{ + { + name: "empty name", + cfg: cfg, + mounts: []ReadOnlyMount{{HostPath: host, Destination: "/workspace/inputs"}}, + }, + { + name: "duplicate name", + cfg: cfg, + mounts: []ReadOnlyMount{{Name: "inputs", HostPath: host, Destination: "/workspace/a"}, {Name: "inputs", HostPath: otherHost, Destination: "/workspace/b"}}, + }, + { + name: "relative host", + cfg: cfg, + mounts: []ReadOnlyMount{{Name: "inputs", HostPath: "relative", Destination: "/workspace/inputs"}}, + }, + { + name: "unclean host", + cfg: cfg, + mounts: []ReadOnlyMount{{Name: "inputs", HostPath: host + string(filepath.Separator) + ".." + string(filepath.Separator) + filepath.Base(host), Destination: "/workspace/inputs"}}, + }, + { + name: "symlink host", + cfg: cfg, + mounts: []ReadOnlyMount{{Name: "inputs", HostPath: symlink, Destination: "/workspace/inputs"}}, + }, + { + name: "missing host", + cfg: cfg, + mounts: []ReadOnlyMount{{Name: "inputs", HostPath: filepath.Join(host, "missing"), Destination: "/workspace/inputs"}}, + }, + { + name: "host file", + cfg: cfg, + mounts: []ReadOnlyMount{{Name: "inputs", HostPath: file, Destination: "/workspace/inputs"}}, + }, + { + name: "relative destination", + cfg: cfg, + mounts: []ReadOnlyMount{{Name: "inputs", HostPath: host, Destination: "inputs"}}, + }, + { + name: "unclean destination", + cfg: cfg, + mounts: []ReadOnlyMount{{Name: "inputs", HostPath: host, Destination: "/workspace/a/../inputs"}}, + }, + { + name: "destination equals workdir", + cfg: cfg, + mounts: []ReadOnlyMount{{Name: "inputs", HostPath: host, Destination: "/workspace"}}, + }, + { + name: "destination sibling prefix", + cfg: cfg, + mounts: []ReadOnlyMount{{Name: "inputs", HostPath: host, Destination: "/workspace-other/inputs"}}, + }, + { + name: "duplicate destination", + cfg: cfg, + mounts: []ReadOnlyMount{{Name: "a", HostPath: host, Destination: "/workspace/inputs"}, {Name: "b", HostPath: otherHost, Destination: "/workspace/inputs"}}, + }, + { + name: "invalid workdir", + cfg: func() Config { + invalid := cfg + invalid.Workdir = "workspace" + return invalid + }(), + mounts: []ReadOnlyMount{{Name: "inputs", HostPath: host, Destination: "/workspace/inputs"}}, + }, + { + name: "workspace access none", + cfg: func() Config { + none := cfg + none.WorkspaceAccess = AccessNone + return none + }(), + mounts: []ReadOnlyMount{{Name: "inputs", HostPath: host, Destination: "/workspace/inputs"}}, + }, + { + name: "destination contains null", + cfg: cfg, + mounts: []ReadOnlyMount{{Name: "inputs", HostPath: host, Destination: "/workspace/\x00inputs"}}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := validateReadOnlyMounts(tt.cfg, tt.mounts) + if err == nil { + t.Fatal("validateReadOnlyMounts() error = nil, want rejection") + } + for _, sensitive := range []string{host, otherHost, file, symlink} { + if strings.Contains(err.Error(), sensitive) { + t.Fatalf("validation error leaked mount root %q: %v", sensitive, err) + } + } + }) + } +} + +func TestAppendDockerMountArgsAddsSortedReadOnlyMounts(t *testing.T) { + workspace := canonicalTempDir(t) + alpha := canonicalTempDir(t) + zeta := canonicalTempDir(t) + cfg := DefaultConfig() + + got, roots, err := appendDockerMountArgs( + context.Background(), + nil, + cfg, + workspace, + []ReadOnlyMount{ + {Name: "zeta", HostPath: zeta, Destination: "/workspace/zeta"}, + {Name: "alpha", HostPath: alpha, Destination: "/workspace/alpha"}, + }, + nil, + ) + if err != nil { + t.Fatalf("appendDockerMountArgs() error = %v", err) + } + + want := []string{ + "-v", workspace + ":/workspace:rw", + "-v", alpha + ":/workspace/alpha:ro", + "-v", zeta + ":/workspace/zeta:ro", + } + if !slices.Equal(got, want) { + t.Fatalf("appendDockerMountArgs() = %#v, want %#v", got, want) + } + for _, root := range []string{workspace, alpha, zeta} { + if !slices.Contains(roots, root) { + t.Fatalf("sensitive roots %#v do not contain %q", roots, root) + } + } +} + +func TestDockerCacheKeyIncludesDeterministicReadOnlyMountSet(t *testing.T) { + cfg := DefaultConfig() + key := "delegation:9db9" + firstHost := canonicalTempDir(t) + secondHost := canonicalTempDir(t) + first := ReadOnlyMount{Name: "alpha", HostPath: firstHost, Destination: "/workspace/alpha"} + second := ReadOnlyMount{Name: "zeta", HostPath: secondHost, Destination: "/workspace/zeta"} + + forward := dockerCacheKey(key, "/workspace/output", cfg, first, second) + reverse := dockerCacheKey(key, "/workspace/output", cfg, second, first) + if forward != reverse { + t.Fatalf("mount ordering changed cache identity: %q != %q", forward, reverse) + } + + changedDestination := first + changedDestination.Destination = "/workspace/other" + if got := dockerCacheKey(key, "/workspace/output", cfg, changedDestination, second); got == forward { + t.Fatalf("destination change did not change cache identity: %q", got) + } + + changedName := first + changedName.Name = "other" + if got := dockerCacheKey(key, "/workspace/output", cfg, changedName, second); got == forward { + t.Fatalf("name change did not change cache identity: %q", got) + } + + changedHost := first + changedHost.HostPath = secondHost + if got := dockerCacheKey(key, "/workspace/output", cfg, changedHost, second); got == forward { + t.Fatalf("host change did not change cache identity: %q", got) + } +} + +func TestWorkspaceAccessOverrideIsPerAcquisition(t *testing.T) { + for _, original := range []Access{AccessNone, AccessRO} { + cfg := DefaultConfig() + cfg.WorkspaceAccess = original + opts := ApplyGetOpts([]GetOption{WithWorkspaceAccessOverride(AccessRW)}) + + got := applyGetOptsToConfig(cfg, opts) + if got.WorkspaceAccess != AccessRW { + t.Fatalf("override from %q = %q, want rw", original, got.WorkspaceAccess) + } + if cfg.WorkspaceAccess != original { + t.Fatalf("override mutated shared config: got %q, want %q", cfg.WorkspaceAccess, original) + } + } +} + +func TestApplyGetOptsCopiesMounts(t *testing.T) { + mounts := []ReadOnlyMount{{Name: "inputs", HostPath: "/canonical/host", Destination: "/workspace/inputs"}} + opts := ApplyGetOpts([]GetOption{WithReadOnlyMounts(mounts...)}) + mounts[0].Name = "mutated" + + if got := opts.ReadOnlyMounts[0].Name; got != "inputs" { + t.Fatalf("ApplyGetOpts retained caller slice: name = %q", got) + } +} + +func TestDockerManagerReleaseRetainsSandboxWhenDestroyFails(t *testing.T) { + destroyErr := errors.New("destroy failed") + sb := &DockerSandbox{ + containerID: "container-1", + destroyFunc: func(context.Context) error { + return destroyErr + }, + } + manager := &DockerManager{sandboxes: map[string]*DockerSandbox{"w123:delegation": sb}} + + err := manager.Release(context.Background(), "delegation") + if !errors.Is(err, destroyErr) { + t.Fatalf("Release() error = %v, want %v", err, destroyErr) + } + if got := manager.Stats()["active"]; got != 1 { + t.Fatalf("active sandboxes after failed release = %v, want 1", got) + } + + sb.destroyFunc = func(context.Context) error { return nil } + if err := manager.Release(context.Background(), "delegation"); err != nil { + t.Fatalf("Release() retry error = %v", err) + } + if got := manager.Stats()["active"]; got != 0 { + t.Fatalf("active sandboxes after successful release = %v, want 0", got) + } +} + +func TestDockerManagerReleaseFailureRemainsVisibleConcurrently(t *testing.T) { + destroyStarted := make(chan struct{}) + allowDestroy := make(chan struct{}) + destroyErr := errors.New("destroy failed") + sb := &DockerSandbox{ + containerID: "container-1", + destroyFunc: func(context.Context) error { + close(destroyStarted) + <-allowDestroy + return destroyErr + }, + } + manager := &DockerManager{sandboxes: map[string]*DockerSandbox{"delegation": sb}} + + releaseDone := make(chan error, 1) + go func() { + releaseDone <- manager.Release(context.Background(), "delegation") + }() + <-destroyStarted + + statsStarted := make(chan struct{}) + statsDone := make(chan map[string]any, 1) + go func() { + close(statsStarted) + statsDone <- manager.Stats() + }() + <-statsStarted + + select { + case <-statsDone: + t.Fatal("Stats returned while Release was still deciding container lifecycle") + case <-time.After(25 * time.Millisecond): + } + + close(allowDestroy) + if err := <-releaseDone; !errors.Is(err, destroyErr) { + t.Fatalf("Release() error = %v, want %v", err, destroyErr) + } + select { + case stats := <-statsDone: + if got := stats["active"]; got != 1 { + t.Fatalf("concurrent active sandboxes = %v, want 1", got) + } + case <-time.After(time.Second): + t.Fatal("Stats did not return after Release completed") + } +} + +func TestRedactMountRootsRemovesSensitivePaths(t *testing.T) { + roots := []string{"/private/runtime/delegation", "/private/runtime/delegation/inputs"} + message := "invalid mount /private/runtime/delegation/inputs at /private/runtime/delegation" + got := redactMountRoots(message, roots) + + for _, root := range roots { + if strings.Contains(got, root) { + t.Fatalf("redactMountRoots() leaked %q in %q", root, got) + } + } + if !strings.Contains(got, redactedMountRoot) { + t.Fatalf("redactMountRoots() = %q, want redaction marker", got) + } +} + +func canonicalTempDir(t *testing.T) string { + t.Helper() + dir := t.TempDir() + canonical, err := filepath.EvalSymlinks(dir) + if err != nil { + t.Fatalf("canonicalize temp dir: %v", err) + } + return canonical +} diff --git a/internal/sandbox/docker_resolve.go b/internal/sandbox/docker_resolve.go index ac425078..d168b5b0 100644 --- a/internal/sandbox/docker_resolve.go +++ b/internal/sandbox/docker_resolve.go @@ -27,7 +27,7 @@ func resolveHostWorkspacePath(ctx context.Context, localPath string) string { containerID := detectContainerID() if containerID == "" { - slog.Error("sandbox.resolve: cannot determine container ID — DooD volume mounts will fail", "path", localPath) + slog.Error("sandbox.resolve: cannot determine container ID — DooD volume mounts will fail") return localPath } @@ -60,10 +60,7 @@ func resolveHostWorkspacePath(ctx context.Context, localPath string) string { if resolved, err := filepath.EvalSymlinks(localPath); err == nil { resolvedClean := filepath.Clean(resolved) if resolvedClean != targetDir { - slog.Info("sandbox.resolve: symlink resolved", - "path", localPath, - "resolved", resolvedClean, - ) + slog.Debug("sandbox.resolve: canonicalized local mount path") } targetDir = resolvedClean } @@ -88,7 +85,7 @@ func resolveHostWorkspacePath(ctx context.Context, localPath string) string { } if bestDest == "" { - slog.Warn("sandbox.resolve: no matching mount found", "path", localPath, "container", containerID) + slog.Warn("sandbox.resolve: no matching mount found", "container", containerID) return localPath } @@ -96,7 +93,7 @@ func resolveHostWorkspacePath(ctx context.Context, localPath string) string { // use the host source path (assumes local volume driver). if bestType == "volume" && bestName != "" { if bestRel == "." { - slog.Debug("sandbox.resolve: resolved to named volume", "path", localPath, "volume", bestName) + slog.Debug("sandbox.resolve: resolved mount to named volume") return bestName } if bestSource != "" { @@ -107,11 +104,11 @@ func resolveHostWorkspacePath(ctx context.Context, localPath string) string { // Bind mount: join source with relative path. if bestSource != "" { resolved := filepath.Join(bestSource, bestRel) - slog.Debug("sandbox.resolve: resolved to host path", "path", localPath, "host", resolved) + slog.Debug("sandbox.resolve: resolved mount to host path") return resolved } - slog.Warn("sandbox.resolve: mount found but no source path", "path", localPath, "mount", bestDest) + slog.Warn("sandbox.resolve: mount found but no source path") return localPath } diff --git a/internal/sandbox/sandbox.go b/internal/sandbox/sandbox.go index f97e5078..a1079761 100644 --- a/internal/sandbox/sandbox.go +++ b/internal/sandbox/sandbox.go @@ -181,6 +181,61 @@ func ApplyExecOpts(opts []ExecOption) ExecOpts { return o } +// ReadOnlyMount describes an additional runtime-only bind mount. +// HostPath must be an absolute, canonical host path. Destination must be an +// absolute canonical container path strictly beneath the configured workdir. +type ReadOnlyMount struct { + Name string + HostPath string + Destination string +} + +// GetOption configures optional behavior for Manager.Get calls. +type GetOption func(*GetOpts) + +// GetOpts holds optional settings applied via GetOption. +type GetOpts struct { + ReadOnlyMounts []ReadOnlyMount + WorkspaceAccessOverride *Access +} + +// WithReadOnlyMounts adds runtime-only read-only mounts to a sandbox. +func WithReadOnlyMounts(mounts ...ReadOnlyMount) GetOption { + copied := append([]ReadOnlyMount(nil), mounts...) + return func(o *GetOpts) { + o.ReadOnlyMounts = append(o.ReadOnlyMounts, copied...) + } +} + +// WithWorkspaceAccessOverride changes access for this isolated sandbox +// instance only. Callers must validate the workspace authority before using it. +func WithWorkspaceAccessOverride(access Access) GetOption { + return func(o *GetOpts) { + override := access + o.WorkspaceAccessOverride = &override + } +} + +// ApplyGetOpts resolves variadic GetOption values without retaining caller-owned +// slices. +func ApplyGetOpts(opts []GetOption) GetOpts { + var o GetOpts + for _, opt := range opts { + if opt != nil { + opt(&o) + } + } + o.ReadOnlyMounts = append([]ReadOnlyMount(nil), o.ReadOnlyMounts...) + return o +} + +func applyGetOptsToConfig(cfg Config, opts GetOpts) Config { + if opts.WorkspaceAccessOverride != nil { + cfg.WorkspaceAccess = *opts.WorkspaceAccessOverride + } + return cfg +} + // Sandbox is the interface for sandboxed code execution. type Sandbox interface { // Exec runs a command inside the sandbox and returns the result. @@ -203,7 +258,7 @@ type Manager interface { // For agent scope: key = agentID // For shared scope: key = "shared" // If cfgOverride is non-nil, it is used instead of the global config for new containers. - Get(ctx context.Context, key string, workspace string, cfgOverride *Config) (Sandbox, error) + Get(ctx context.Context, key string, workspace string, cfgOverride *Config, opts ...GetOption) (Sandbox, error) // Release destroys a sandbox by key. Release(ctx context.Context, key string) error diff --git a/internal/store/agent_link_store.go b/internal/store/agent_link_store.go index fd86e2f3..285c9cac 100644 --- a/internal/store/agent_link_store.go +++ b/internal/store/agent_link_store.go @@ -25,7 +25,7 @@ type AgentLinkData struct { BaseModel SourceAgentID uuid.UUID `json:"source_agent_id" db:"source_agent_id"` TargetAgentID uuid.UUID `json:"target_agent_id" db:"target_agent_id"` - Direction string `json:"direction" db:"direction"` // "outbound", "inbound", "bidirectional" + Direction string `json:"direction" db:"direction"` // "outbound", "inbound", "bidirectional" TeamID *uuid.UUID `json:"team_id,omitempty" db:"team_id"` // non-nil = auto-created by team Description string `json:"description,omitempty" db:"description"` MaxConcurrent int `json:"max_concurrent" db:"max_concurrent"` @@ -34,13 +34,13 @@ type AgentLinkData struct { CreatedBy string `json:"created_by" db:"created_by"` // Joined fields (populated by queries that JOIN agents table) - SourceAgentKey string `json:"source_agent_key,omitempty" db:"source_agent_key"` - SourceDisplayName string `json:"source_display_name,omitempty" db:"source_display_name"` - SourceEmoji string `json:"source_emoji,omitempty" db:"source_emoji"` - TargetAgentKey string `json:"target_agent_key,omitempty" db:"target_agent_key"` - TargetDisplayName string `json:"target_display_name,omitempty" db:"target_display_name"` - TargetEmoji string `json:"target_emoji,omitempty" db:"target_emoji"` - TargetDescription string `json:"target_description,omitempty" db:"target_description"` + SourceAgentKey string `json:"source_agent_key,omitempty" db:"source_agent_key"` + SourceDisplayName string `json:"source_display_name,omitempty" db:"source_display_name"` + SourceEmoji string `json:"source_emoji,omitempty" db:"source_emoji"` + TargetAgentKey string `json:"target_agent_key,omitempty" db:"target_agent_key"` + TargetDisplayName string `json:"target_display_name,omitempty" db:"target_display_name"` + TargetEmoji string `json:"target_emoji,omitempty" db:"target_emoji"` + TargetDescription string `json:"target_description,omitempty" db:"target_description"` TeamName string `json:"team_name,omitempty" db:"team_name"` // from LEFT JOIN agent_teams (link's own team) TargetIsTeamLead bool `json:"target_is_team_lead,omitempty" db:"target_is_team_lead"` // true if target is lead of any active team TargetTeamName string `json:"target_team_name,omitempty" db:"target_team_name"` // name of team the target leads @@ -62,7 +62,9 @@ type AgentLinkStore interface { // CanDelegate checks if fromAgent can delegate to toAgent considering direction. CanDelegate(ctx context.Context, fromAgentID, toAgentID uuid.UUID) (bool, error) - // GetLinkBetween returns the active link allowing fromAgent to delegate to toAgent. + // GetLinkBetween returns the active directional link allowing fromAgent to + // delegate to toAgent. MaxConcurrent is retained for API compatibility but + // is reserved and has no runtime admission effect. // Returns full link data including Settings for per-user permission checks. // Returns nil, nil if no matching link exists. GetLinkBetween(ctx context.Context, fromAgentID, toAgentID uuid.UUID) (*AgentLinkData, error) diff --git a/internal/store/pg/agent_links.go b/internal/store/pg/agent_links.go index d345f1de..f809fc80 100644 --- a/internal/store/pg/agent_links.go +++ b/internal/store/pg/agent_links.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "encoding/json" + "errors" "fmt" "time" @@ -257,10 +258,15 @@ func (s *PGAgentLinkStore) GetLinkBetween(ctx context.Context, fromAgentID, toAg (source_agent_id = $1 AND target_agent_id = $2 AND direction IN ('outbound', 'bidirectional')) OR (source_agent_id = $2 AND target_agent_id = $1 AND direction IN ('inbound', 'bidirectional')) - ) LIMIT 1`, args...) + ) + ORDER BY CASE WHEN source_agent_id = $1 THEN 0 ELSE 1 END, created_at, id + LIMIT 1`, args...) d, err := scanLinkRow(row) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } if err != nil { - return nil, nil // no link found + return nil, err } return d, nil } diff --git a/internal/store/pg/factory.go b/internal/store/pg/factory.go index 578c7601..cb5a2880 100644 --- a/internal/store/pg/factory.go +++ b/internal/store/pg/factory.go @@ -22,6 +22,7 @@ func NewPGStores(cfg store.StoreConfig) (*store.Stores, error) { if skillsDir == "" { skillsDir = config.ResolvedDataDirFromEnv() + "/skills-store" } + subagentTasks := NewPGSubagentTaskStore(db) pgStores := &store.Stores{ DB: db, @@ -59,7 +60,8 @@ func NewPGStores(cfg store.StoreConfig) (*store.Stores, error) { SkillTenantCfgs: NewPGSkillTenantConfigStore(db), SkillEvolution: NewPGSkillEvolutionStore(db), SystemConfigs: NewPGSystemConfigStore(db), - SubagentTasks: NewPGSubagentTaskStore(db), + SubagentTasks: subagentTasks, + SubagentTaskRecovery: subagentTasks, Vault: NewPGVaultStore(db), Episodic: NewPGEpisodicStore(db), EvolutionMetrics: NewPGEvolutionMetricsStore(db), diff --git a/internal/store/pg/subagent_tasks.go b/internal/store/pg/subagent_tasks.go index 70cc801b..97b84856 100644 --- a/internal/store/pg/subagent_tasks.go +++ b/internal/store/pg/subagent_tasks.go @@ -18,18 +18,23 @@ type PGSubagentTaskStore struct { db *sql.DB } +var _ store.SubagentTaskRecoveryStore = (*PGSubagentTaskStore)(nil) + // NewPGSubagentTaskStore creates a new PostgreSQL-backed subagent task store. func NewPGSubagentTaskStore(db *sql.DB) *PGSubagentTaskStore { return &PGSubagentTaskStore{db: db} } -const subagentTaskInsertCols = `tenant_id, parent_agent_key, session_key, subject, description, +const subagentTaskInsertCols = `tenant_id, root_agent_id, parent_agent_key, session_key, subject, description, status, result, depth, model, provider, iterations, input_tokens, output_tokens, origin_channel, origin_chat_id, origin_peer_kind, origin_user_id, spawned_by, metadata` // Create persists a new subagent task at spawn time. func (s *PGSubagentTaskStore) Create(ctx context.Context, task *store.SubagentTaskData) error { tid := tenantIDForInsert(ctx) + if task.RootAgentID == uuid.Nil { + return store.ErrSubagentRootAgentIDRequired + } metaJSON := []byte("{}") if len(task.Metadata) > 0 { @@ -39,11 +44,11 @@ func (s *PGSubagentTaskStore) Create(ctx context.Context, task *store.SubagentTa } q := fmt.Sprintf(`INSERT INTO subagent_tasks (id, %s) - VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21) ON CONFLICT (id) DO NOTHING`, subagentTaskInsertCols) _, err := s.db.ExecContext(ctx, q, - task.ID, tid, task.ParentAgentKey, task.SessionKey, task.Subject, task.Description, + task.ID, tid, task.RootAgentID, task.ParentAgentKey, task.SessionKey, task.Subject, task.Description, task.Status, task.Result, task.Depth, task.Model, task.Provider, task.Iterations, task.InputTokens, task.OutputTokens, task.OriginChannel, task.OriginChatID, task.OriginPeerKind, task.OriginUserID, @@ -52,7 +57,7 @@ func (s *PGSubagentTaskStore) Create(ctx context.Context, task *store.SubagentTa return err } -const subagentTaskSelectCols = `id, tenant_id, parent_agent_key, session_key, subject, description, +const subagentTaskSelectCols = `id, tenant_id, root_agent_id, parent_agent_key, session_key, subject, description, status, result, depth, model, provider, iterations, input_tokens, output_tokens, origin_channel, origin_chat_id, origin_peer_kind, origin_user_id, spawned_by, completed_at, archived_at, COALESCE(metadata, '{}'), created_at, updated_at` @@ -62,7 +67,7 @@ func scanTask(row interface{ Scan(...any) error }) (*store.SubagentTaskData, err var t store.SubagentTaskData var metaJSON []byte err := row.Scan( - &t.ID, &t.TenantID, &t.ParentAgentKey, &t.SessionKey, &t.Subject, &t.Description, + &t.ID, &t.TenantID, &t.RootAgentID, &t.ParentAgentKey, &t.SessionKey, &t.Subject, &t.Description, &t.Status, &t.Result, &t.Depth, &t.Model, &t.Provider, &t.Iterations, &t.InputTokens, &t.OutputTokens, &t.OriginChannel, &t.OriginChatID, &t.OriginPeerKind, &t.OriginUserID, &t.SpawnedBy, @@ -77,14 +82,20 @@ func scanTask(row interface{ Scan(...any) error }) (*store.SubagentTaskData, err return &t, nil } -// Get retrieves a single task by ID (tenant-scoped). -func (s *PGSubagentTaskStore) Get(ctx context.Context, id uuid.UUID) (*store.SubagentTaskData, error) { +// Get retrieves a task owned by the tenant and immutable root-agent UUID. +func (s *PGSubagentTaskStore) Get( + ctx context.Context, rootAgentID, id uuid.UUID, +) (*store.SubagentTaskData, error) { tid, err := requireTenantID(ctx) if err != nil { return nil, err } - q := fmt.Sprintf(`SELECT %s FROM subagent_tasks WHERE id = $1 AND tenant_id = $2`, subagentTaskSelectCols) - row := s.db.QueryRowContext(ctx, q, id, tid) + if rootAgentID == uuid.Nil { + return nil, store.ErrSubagentRootAgentIDRequired + } + q := fmt.Sprintf(`SELECT %s FROM subagent_tasks + WHERE id = $1 AND tenant_id = $2 AND root_agent_id = $3`, subagentTaskSelectCols) + row := s.db.QueryRowContext(ctx, q, id, tid, rootAgentID) t, err := scanTask(row) if errors.Is(err, sql.ErrNoRows) { return nil, nil @@ -94,7 +105,7 @@ func (s *PGSubagentTaskStore) Get(ctx context.Context, id uuid.UUID) (*store.Sub // UpdateStatus updates status, result, iterations, and token counts. func (s *PGSubagentTaskStore) UpdateStatus( - ctx context.Context, id uuid.UUID, + ctx context.Context, rootAgentID, id uuid.UUID, status string, result *string, iterations int, inputTokens, outputTokens int64, ) error { @@ -102,9 +113,12 @@ func (s *PGSubagentTaskStore) UpdateStatus( if err != nil { return err } + if rootAgentID == uuid.Nil { + return store.ErrSubagentRootAgentIDRequired + } var completedAt *time.Time - if status != "running" { + if store.IsTerminalSubagentTaskStatus(status) { now := time.Now().UTC() completedAt = &now } @@ -113,34 +127,49 @@ func (s *PGSubagentTaskStore) UpdateStatus( status = $1, result = $2, iterations = $3, input_tokens = $4, output_tokens = $5, completed_at = $6, updated_at = NOW() - WHERE id = $7 AND tenant_id = $8` - _, err = s.db.ExecContext(ctx, q, + WHERE id = $7 AND tenant_id = $8 AND root_agent_id = $9` + res, err := s.db.ExecContext(ctx, q, status, result, iterations, inputTokens, outputTokens, - completedAt, id, tid, + completedAt, id, tid, rootAgentID, ) - return err + if err != nil { + return err + } + affected, err := res.RowsAffected() + if err != nil { + return err + } + if affected != 1 { + return store.ErrSubagentTaskNotFound + } + return nil } -// ListByParent returns tasks for a parent agent key, optionally filtered by status. +// ListByParent returns tasks for a root-agent UUID, optionally filtered by status. func (s *PGSubagentTaskStore) ListByParent( - ctx context.Context, parentAgentKey string, statusFilter string, + ctx context.Context, rootAgentID uuid.UUID, statusFilter string, ) ([]store.SubagentTaskData, error) { tid, err := requireTenantID(ctx) if err != nil { return nil, err } + if rootAgentID == uuid.Nil { + return nil, store.ErrSubagentRootAgentIDRequired + } var rows *sql.Rows if statusFilter != "" { q := fmt.Sprintf(`SELECT %s FROM subagent_tasks - WHERE tenant_id = $1 AND parent_agent_key = $2 AND status = $3 - ORDER BY created_at DESC LIMIT 50`, subagentTaskSelectCols) - rows, err = s.db.QueryContext(ctx, q, tid, parentAgentKey, statusFilter) + WHERE tenant_id = $1 AND root_agent_id = $2 AND status = $3 + AND COALESCE(metadata->>'completion_kind', 'subagent') <> 'delegate' + ORDER BY created_at DESC LIMIT 50`, subagentTaskSelectCols) + rows, err = s.db.QueryContext(ctx, q, tid, rootAgentID, statusFilter) } else { q := fmt.Sprintf(`SELECT %s FROM subagent_tasks - WHERE tenant_id = $1 AND parent_agent_key = $2 - ORDER BY created_at DESC LIMIT 50`, subagentTaskSelectCols) - rows, err = s.db.QueryContext(ctx, q, tid, parentAgentKey) + WHERE tenant_id = $1 AND root_agent_id = $2 + AND COALESCE(metadata->>'completion_kind', 'subagent') <> 'delegate' + ORDER BY created_at DESC LIMIT 50`, subagentTaskSelectCols) + rows, err = s.db.QueryContext(ctx, q, tid, rootAgentID) } if err != nil { return nil, err @@ -152,17 +181,21 @@ func (s *PGSubagentTaskStore) ListByParent( // ListBySession returns tasks for a specific session key (tenant-scoped). func (s *PGSubagentTaskStore) ListBySession( - ctx context.Context, sessionKey string, + ctx context.Context, rootAgentID uuid.UUID, sessionKey string, ) ([]store.SubagentTaskData, error) { tid, err := requireTenantID(ctx) if err != nil { return nil, err } + if rootAgentID == uuid.Nil { + return nil, store.ErrSubagentRootAgentIDRequired + } q := fmt.Sprintf(`SELECT %s FROM subagent_tasks - WHERE tenant_id = $1 AND session_key = $2 + WHERE tenant_id = $1 AND root_agent_id = $2 AND session_key = $3 + AND COALESCE(metadata->>'completion_kind', 'subagent') <> 'delegate' ORDER BY created_at DESC LIMIT 50`, subagentTaskSelectCols) - rows, err := s.db.QueryContext(ctx, q, tid, sessionKey) + rows, err := s.db.QueryContext(ctx, q, tid, rootAgentID, sessionKey) if err != nil { return nil, err } @@ -171,25 +204,54 @@ func (s *PGSubagentTaskStore) ListBySession( return collectTasks(rows) } -// Archive marks old completed/failed/cancelled tasks as archived. -func (s *PGSubagentTaskStore) Archive(ctx context.Context, olderThan time.Duration) (int64, error) { +// Archive marks a bounded batch of old terminal tasks as archived. +func (s *PGSubagentTaskStore) Archive( + ctx context.Context, rootAgentID uuid.UUID, olderThan time.Duration, limit int, +) (int64, error) { + tid, err := requireTenantID(ctx) + if err != nil { + return 0, err + } + if rootAgentID == uuid.Nil { + return 0, store.ErrSubagentRootAgentIDRequired + } + if limit <= 0 { + return 0, nil + } + cutoff := time.Now().UTC().Add(-olderThan) - q := `UPDATE subagent_tasks SET archived_at = NOW(), updated_at = NOW() - WHERE status IN ('completed', 'failed', 'cancelled') - AND archived_at IS NULL AND completed_at < $1` - res, err := s.db.ExecContext(ctx, q, cutoff) + q := `WITH candidates AS ( + SELECT id + FROM subagent_tasks + WHERE tenant_id = $1 AND root_agent_id = $2 + AND status IN ('completed', 'failed', 'cancelled') + AND archived_at IS NULL AND completed_at < $3 + ORDER BY completed_at, id + LIMIT $4 + FOR UPDATE SKIP LOCKED + ) + UPDATE subagent_tasks AS task + SET archived_at = NOW(), updated_at = NOW() + FROM candidates + WHERE task.id = candidates.id` + res, err := s.db.ExecContext(ctx, q, tid, rootAgentID, cutoff, limit) if err != nil { return 0, err } return res.RowsAffected() } -// UpdateMetadata merges metadata on an existing task. -func (s *PGSubagentTaskStore) UpdateMetadata(ctx context.Context, id uuid.UUID, metadata map[string]any) error { +// UpdateMetadata merges metadata on a task owned by the tenant and root agent. +func (s *PGSubagentTaskStore) UpdateMetadata( + ctx context.Context, rootAgentID, id uuid.UUID, metadata map[string]any, +) error { tid, err := requireTenantID(ctx) if err != nil { return err } + if rootAgentID == uuid.Nil { + return store.ErrSubagentRootAgentIDRequired + } metaJSON, err := json.Marshal(metadata) if err != nil { @@ -197,9 +259,35 @@ func (s *PGSubagentTaskStore) UpdateMetadata(ctx context.Context, id uuid.UUID, } q := `UPDATE subagent_tasks SET metadata = metadata || $1, updated_at = NOW() - WHERE id = $2 AND tenant_id = $3` - _, err = s.db.ExecContext(ctx, q, metaJSON, id, tid) - return err + WHERE id = $2 AND tenant_id = $3 AND root_agent_id = $4` + res, err := s.db.ExecContext(ctx, q, metaJSON, id, tid, rootAgentID) + if err != nil { + return err + } + affected, err := res.RowsAffected() + if err != nil { + return err + } + if affected != 1 { + return store.ErrSubagentTaskNotFound + } + return nil +} + +// RecoverInterrupted marks child runs that cannot survive a process restart as +// failed. This cross-tenant update runs before the gateway accepts new work. +func (s *PGSubagentTaskStore) RecoverInterrupted(ctx context.Context) (int64, error) { + if !store.IsMasterScope(ctx) { + return 0, fmt.Errorf("recover interrupted subagent tasks requires master scope") + } + q := `UPDATE subagent_tasks SET + status = 'failed', result = $1, completed_at = NOW(), updated_at = NOW() + WHERE status NOT IN ('completed', 'failed', 'cancelled')` + res, err := s.db.ExecContext(ctx, q, store.InterruptedSubagentTaskResult) + if err != nil { + return 0, err + } + return res.RowsAffected() } // collectTasks scans rows into a slice. diff --git a/internal/store/pg/subagent_tasks_test.go b/internal/store/pg/subagent_tasks_test.go new file mode 100644 index 00000000..ca5ac36f --- /dev/null +++ b/internal/store/pg/subagent_tasks_test.go @@ -0,0 +1,472 @@ +package pg + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/google/uuid" + + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +func createPGSubagentTask( + t *testing.T, + taskStore *PGSubagentTaskStore, + ctx context.Context, + rootAgentID uuid.UUID, + rootAgentKey, sessionKey, status string, +) uuid.UUID { + t.Helper() + + id := uuid.Must(uuid.NewV7()) + task := &store.SubagentTaskData{ + RootAgentID: rootAgentID, + ParentAgentKey: rootAgentKey, + SessionKey: &sessionKey, + Subject: "store test", + Description: "verify scoped persistence", + Status: status, + Depth: 1, + Metadata: map[string]any{}, + } + task.ID = id + if err := taskStore.Create(ctx, task); err != nil { + t.Fatalf("Create(%s): %v", id, err) + } + return id +} + +func TestPGSubagentTaskStoreRequiresTenantAndRootScope(t *testing.T) { + db := hooksTestDB(t) + tenantA, rootAID := seedTenantAndAgent(t, db) + tenantB, tenantBRootID := seedTenantAndAgent(t, db) + ctxA := tenantScopedCtx(tenantA) + ctxB := tenantScopedCtx(tenantB) + taskStore := NewPGSubagentTaskStore(db) + + const ( + rootA = "root-a" + rootB = "root-b" + sessionID = "shared-session" + ) + rootBID := uuid.Must(uuid.NewV7()) + if _, err := db.Exec( + `INSERT INTO agents (id, tenant_id, agent_key, agent_type, status, provider, model, owner_id) + VALUES ($1,$2,$3,'predefined','active','test','test-model','owner')`, + rootBID, tenantA, rootB, + ); err != nil { + t.Fatalf("seed second root agent: %v", err) + } + taskA := createPGSubagentTask(t, taskStore, ctxA, rootAID, rootA, sessionID, "queued") + taskB := createPGSubagentTask(t, taskStore, ctxA, rootBID, rootB, sessionID, "queued") + delegationTask := createPGSubagentTask(t, taskStore, ctxA, rootAID, rootA, sessionID, "queued") + if err := taskStore.UpdateMetadata(ctxA, rootAID, delegationTask, map[string]any{ + "completion_kind": "delegate", + }); err != nil { + t.Fatalf("mark delegation completion: %v", err) + } + _ = createPGSubagentTask(t, taskStore, ctxB, tenantBRootID, rootA, sessionID, "queued") + crossTenantTask := &store.SubagentTaskData{ + BaseModel: store.BaseModel{ID: uuid.Must(uuid.NewV7())}, + RootAgentID: tenantBRootID, + ParentAgentKey: rootA, + Subject: "cross tenant", + Description: "must be rejected", + Status: "queued", + Depth: 1, + } + if err := taskStore.Create(ctxA, crossTenantTask); err == nil { + t.Fatal("Create accepted root agent from another tenant") + } + + got, err := taskStore.Get(ctxA, rootAID, taskA) + if err != nil { + t.Fatalf("Get owning scope: %v", err) + } + if got == nil || got.ID != taskA { + t.Fatalf("Get owning scope = %#v, want task %s", got, taskA) + } + + got, err = taskStore.Get(ctxA, rootBID, taskA) + if err != nil { + t.Fatalf("Get cross-root: %v", err) + } + if got != nil { + t.Fatalf("Get cross-root = %#v, want nil", got) + } + + got, err = taskStore.Get(ctxB, rootAID, taskA) + if err != nil { + t.Fatalf("Get cross-tenant: %v", err) + } + if got != nil { + t.Fatalf("Get cross-tenant = %#v, want nil", got) + } + + if err := taskStore.UpdateStatus(ctxA, rootBID, taskA, "completed", nil, 3, 10, 20); !errors.Is(err, store.ErrSubagentTaskNotFound) { + t.Fatalf("UpdateStatus cross-root error = %v, want scoped not found", err) + } + got, err = taskStore.Get(ctxA, rootAID, taskA) + if err != nil { + t.Fatalf("Get after cross-root status update: %v", err) + } + if got.Status != "queued" || got.CompletedAt != nil { + t.Fatalf("cross-root status update changed task: status=%q completed_at=%v", got.Status, got.CompletedAt) + } + + if err := taskStore.UpdateMetadata(ctxA, rootBID, taskA, map[string]any{"denied": true}); !errors.Is(err, store.ErrSubagentTaskNotFound) { + t.Fatalf("UpdateMetadata cross-root error = %v, want scoped not found", err) + } + got, err = taskStore.Get(ctxA, rootAID, taskA) + if err != nil { + t.Fatalf("Get after cross-root metadata update: %v", err) + } + if _, exists := got.Metadata["denied"]; exists { + t.Fatalf("cross-root metadata update changed task: %#v", got.Metadata) + } + if err := taskStore.UpdateMetadata(ctxA, rootAID, taskA, map[string]any{ + "announcement_status": "undelivered", + "delivered": false, + }); err != nil { + t.Fatalf("UpdateMetadata owning scope: %v", err) + } + got, err = taskStore.Get(ctxA, rootAID, taskA) + if err != nil { + t.Fatalf("Get after owning metadata update: %v", err) + } + if got.Metadata["announcement_status"] != "undelivered" || got.Metadata["delivered"] != false { + t.Fatalf("metadata JSON types were not preserved: %#v", got.Metadata) + } + + parentTasksA, err := taskStore.ListByParent(ctxA, rootAID, "") + if err != nil { + t.Fatalf("ListByParent root A: %v", err) + } + if len(parentTasksA) != 1 || parentTasksA[0].ID != taskA { + t.Fatalf("ListByParent root A = %#v, want only self-clone %s", parentTasksA, taskA) + } + parentTasksB, err := taskStore.ListByParent(ctxA, rootBID, "queued") + if err != nil { + t.Fatalf("ListByParent root B: %v", err) + } + if len(parentTasksB) != 1 || parentTasksB[0].ID != taskB { + t.Fatalf("ListByParent root B = %#v, want only %s", parentTasksB, taskB) + } + + tasksA, err := taskStore.ListBySession(ctxA, rootAID, sessionID) + if err != nil { + t.Fatalf("ListBySession root A: %v", err) + } + if len(tasksA) != 1 || tasksA[0].ID != taskA { + t.Fatalf("ListBySession root A = %#v, want only self-clone %s", tasksA, taskA) + } + tasksB, err := taskStore.ListBySession(ctxA, rootBID, sessionID) + if err != nil { + t.Fatalf("ListBySession root B: %v", err) + } + if len(tasksB) != 1 || tasksB[0].ID != taskB { + t.Fatalf("ListBySession root B = %#v, want only %s", tasksB, taskB) + } + + if _, err := taskStore.Get(context.Background(), rootAID, taskA); err == nil { + t.Fatal("Get without tenant context returned nil error") + } + if _, err := taskStore.Get(ctxA, uuid.Nil, taskA); !errors.Is(err, store.ErrSubagentRootAgentIDRequired) { + t.Fatalf("Get empty root error = %v, want %v", err, store.ErrSubagentRootAgentIDRequired) + } +} + +func TestPGSubagentTaskStoreRejectsRecreatedAgentWithSameKey(t *testing.T) { + db := hooksTestDB(t) + tenantID, oldRootAgentID := seedTenantAndAgent(t, db) + ctx := tenantScopedCtx(tenantID) + taskStore := NewPGSubagentTaskStore(db) + + const rootAgentKey = "recreated-root" + if _, err := db.Exec( + `UPDATE agents SET agent_key = $1 WHERE id = $2`, + rootAgentKey, oldRootAgentID, + ); err != nil { + t.Fatalf("rename original root agent: %v", err) + } + taskID := createPGSubagentTask( + t, taskStore, ctx, oldRootAgentID, rootAgentKey, "recreated-root-session", "queued", + ) + + if _, err := db.Exec(`DELETE FROM agents WHERE id = $1`, oldRootAgentID); err != nil { + t.Fatalf("delete original root agent: %v", err) + } + newRootAgentID := uuid.Must(uuid.NewV7()) + if _, err := db.Exec( + `INSERT INTO agents (id, tenant_id, agent_key, agent_type, status, provider, model, owner_id) + VALUES ($1,$2,$3,'predefined','active','test','test-model','owner')`, + newRootAgentID, tenantID, rootAgentKey, + ); err != nil { + t.Fatalf("recreate root agent: %v", err) + } + + got, err := taskStore.Get(ctx, newRootAgentID, taskID) + if err != nil { + t.Fatalf("Get with recreated root agent: %v", err) + } + if got != nil { + t.Fatalf("Get with recreated root agent returned old task %s", got.ID) + } + if err := taskStore.UpdateStatus(ctx, newRootAgentID, taskID, "completed", nil, 1, 2, 3); !errors.Is(err, store.ErrSubagentTaskNotFound) { + t.Fatalf("UpdateStatus with recreated root agent error = %v, want scoped not found", err) + } + var status string + var rootAgentID uuid.NullUUID + if err := db.QueryRow( + `SELECT status, root_agent_id FROM subagent_tasks WHERE id = $1`, taskID, + ).Scan(&status, &rootAgentID); err != nil { + t.Fatalf("read preserved old task: %v", err) + } + if status != "queued" || rootAgentID.Valid { + t.Fatalf("old task after recreate: status=%q root_agent_id=%s, want queued/NULL", status, rootAgentID.UUID) + } +} + +func TestPGSubagentTaskStoreCompletedAtOnlyForTerminalStatus(t *testing.T) { + db := hooksTestDB(t) + tenantID, rootAgentID := seedTenantAndAgent(t, db) + ctx := tenantScopedCtx(tenantID) + taskStore := NewPGSubagentTaskStore(db) + + tests := []struct { + status string + terminal bool + }{ + {status: "new"}, + {status: "queued"}, + {status: "running"}, + {status: "waiting_child"}, + {status: "completed", terminal: true}, + {status: "failed", terminal: true}, + {status: "cancelled", terminal: true}, + } + + for _, tt := range tests { + t.Run(tt.status, func(t *testing.T) { + rootAgentKey := "root-" + tt.status + id := createPGSubagentTask(t, taskStore, ctx, rootAgentID, rootAgentKey, "session-"+tt.status, "queued") + if err := taskStore.UpdateStatus(ctx, rootAgentID, id, tt.status, nil, 0, 0, 0); err != nil { + t.Fatalf("UpdateStatus(%q): %v", tt.status, err) + } + got, err := taskStore.Get(ctx, rootAgentID, id) + if err != nil { + t.Fatalf("Get(%q): %v", tt.status, err) + } + if (got.CompletedAt != nil) != tt.terminal { + t.Fatalf("status %q completed_at = %v, terminal=%v", tt.status, got.CompletedAt, tt.terminal) + } + }) + } +} + +func TestPGSubagentTaskStoreRecoverInterrupted(t *testing.T) { + db := hooksTestDB(t) + // Recovery is intentionally process-global. Reset its shared table so rows + // left by other store tests cannot affect the exact recovery count. + if _, err := db.Exec(`DELETE FROM subagent_tasks`); err != nil { + t.Fatalf("reset subagent tasks before global recovery: %v", err) + } + tenantA, rootAID := seedTenantAndAgent(t, db) + tenantB, rootBID := seedTenantAndAgent(t, db) + ctxA := tenantScopedCtx(tenantA) + ctxB := tenantScopedCtx(tenantB) + taskStore := NewPGSubagentTaskStore(db) + + queuedID := createPGSubagentTask(t, taskStore, ctxA, rootAID, "root-a", "queued", "queued") + if _, err := db.Exec( + `UPDATE subagent_tasks SET completed_at = $1 WHERE id = $2`, + time.Now().UTC().Add(-time.Hour), + queuedID, + ); err != nil { + t.Fatalf("seed malformed queued completed_at: %v", err) + } + runningID := createPGSubagentTask(t, taskStore, ctxA, rootAID, "root-a", "running", "running") + waitingID := createPGSubagentTask( + t, taskStore, ctxB, rootBID, "root-b", "waiting", "waiting_child", + ) + if err := taskStore.UpdateMetadata(ctxB, rootBID, waitingID, map[string]any{ + "completion_kind": "delegate", + "completion_media": []map[string]any{{ + "path": ".delegations/completed-before-crash/report.pdf", + "mime_type": "application/pdf", + }}, + }); err != nil { + t.Fatalf("record published artifact metadata: %v", err) + } + completedID := createPGSubagentTask(t, taskStore, ctxB, rootBID, "root-b", "completed", "queued") + completedResult := "already completed" + if err := taskStore.UpdateStatus( + ctxB, rootBID, completedID, "completed", &completedResult, 1, 2, 3, + ); err != nil { + t.Fatalf("complete terminal task: %v", err) + } + + if _, err := taskStore.RecoverInterrupted(ctxA); err == nil { + t.Fatal("RecoverInterrupted accepted tenant-scoped context") + } + recoveryCtx := store.WithTenantID(context.Background(), store.MasterTenantID) + recovered, err := taskStore.RecoverInterrupted(recoveryCtx) + if err != nil { + t.Fatalf("RecoverInterrupted: %v", err) + } + if recovered != 3 { + t.Fatalf("RecoverInterrupted recovered %d tasks, want 3", recovered) + } + + for _, item := range []struct { + ctx context.Context + root uuid.UUID + id uuid.UUID + }{ + {ctx: ctxA, root: rootAID, id: queuedID}, + {ctx: ctxA, root: rootAID, id: runningID}, + {ctx: ctxB, root: rootBID, id: waitingID}, + } { + got, getErr := taskStore.Get(item.ctx, item.root, item.id) + if getErr != nil { + t.Fatalf("Get recovered task %s: %v", item.id, getErr) + } + if got.Status != "failed" || got.CompletedAt == nil || got.Result == nil || + !strings.Contains(*got.Result, "gateway stopped") { + t.Fatalf("recovered task %s = %#v", item.id, got) + } + if item.id == waitingID { + if got.Metadata["completion_kind"] != "delegate" || + got.Metadata["completion_media"] == nil { + t.Fatalf("published artifact metadata was lost: %#v", got.Metadata) + } + } + } + + completed, err := taskStore.Get(ctxB, rootBID, completedID) + if err != nil { + t.Fatalf("Get completed task: %v", err) + } + if completed.Status != "completed" || completed.Result == nil || + *completed.Result != completedResult { + t.Fatalf("completed task changed during recovery: %#v", completed) + } + + recovered, err = taskStore.RecoverInterrupted(recoveryCtx) + if err != nil { + t.Fatalf("RecoverInterrupted second pass: %v", err) + } + if recovered != 0 { + t.Fatalf("RecoverInterrupted second pass recovered %d tasks, want 0", recovered) + } +} + +func TestPGSubagentTaskStoreArchiveIsScopedAndBounded(t *testing.T) { + db := hooksTestDB(t) + tenantA, rootAID := seedTenantAndAgent(t, db) + tenantB, tenantBRootID := seedTenantAndAgent(t, db) + ctxA := tenantScopedCtx(tenantA) + ctxB := tenantScopedCtx(tenantB) + taskStore := NewPGSubagentTaskStore(db) + + const ( + rootA = "root-a" + rootB = "root-b" + ) + rootBID := uuid.Must(uuid.NewV7()) + if _, err := db.Exec( + `INSERT INTO agents (id, tenant_id, agent_key, agent_type, status, provider, model, owner_id) + VALUES ($1,$2,$3,'predefined','active','test','test-model','owner')`, + rootBID, tenantA, rootB, + ); err != nil { + t.Fatalf("seed second root agent: %v", err) + } + var rootATasks []uuid.UUID + for i := range 3 { + rootATasks = append(rootATasks, createPGSubagentTask( + t, taskStore, ctxA, rootAID, rootA, fmt.Sprintf("session-a-%d", i), "queued", + )) + } + rootBTask := createPGSubagentTask(t, taskStore, ctxA, rootBID, rootB, "session-b", "queued") + tenantBTask := createPGSubagentTask(t, taskStore, ctxB, tenantBRootID, rootA, "session-other-tenant", "queued") + queuedTask := createPGSubagentTask(t, taskStore, ctxA, rootAID, rootA, "session-queued", "queued") + + for _, item := range []struct { + ctx context.Context + root uuid.UUID + id uuid.UUID + }{ + {ctx: ctxA, root: rootAID, id: rootATasks[0]}, + {ctx: ctxA, root: rootAID, id: rootATasks[1]}, + {ctx: ctxA, root: rootAID, id: rootATasks[2]}, + {ctx: ctxA, root: rootBID, id: rootBTask}, + {ctx: ctxB, root: tenantBRootID, id: tenantBTask}, + } { + if err := taskStore.UpdateStatus(item.ctx, item.root, item.id, "completed", nil, 0, 0, 0); err != nil { + t.Fatalf("UpdateStatus(%s): %v", item.id, err) + } + } + + oldTime := time.Now().UTC().Add(-2 * time.Hour) + for _, id := range append(append([]uuid.UUID{}, rootATasks...), rootBTask, tenantBTask) { + if _, err := db.Exec(`UPDATE subagent_tasks SET completed_at = $1 WHERE id = $2`, oldTime, id); err != nil { + t.Fatalf("backdate terminal task %s: %v", id, err) + } + } + // A non-terminal row must remain unarchived even if malformed legacy data + // happens to carry a completed_at value. + if _, err := db.Exec(`UPDATE subagent_tasks SET completed_at = $1 WHERE id = $2`, oldTime, queuedTask); err != nil { + t.Fatalf("backdate queued task: %v", err) + } + + archived, err := taskStore.Archive(ctxA, rootAID, time.Hour, 2) + if err != nil { + t.Fatalf("Archive first batch: %v", err) + } + if archived != 2 { + t.Fatalf("Archive first batch affected %d rows, want 2", archived) + } + + assertPGArchivedCount(t, db, tenantA, rootAID, 2) + assertPGArchivedCount(t, db, tenantA, rootBID, 0) + assertPGArchivedCount(t, db, tenantB, tenantBRootID, 0) + + archived, err = taskStore.Archive(ctxA, rootAID, time.Hour, 2) + if err != nil { + t.Fatalf("Archive second batch: %v", err) + } + if archived != 1 { + t.Fatalf("Archive second batch affected %d rows, want 1", archived) + } + assertPGArchivedCount(t, db, tenantA, rootAID, 3) + + var queuedArchived sql.NullTime + if err := db.QueryRow(`SELECT archived_at FROM subagent_tasks WHERE id = $1`, queuedTask).Scan(&queuedArchived); err != nil { + t.Fatalf("read queued archived_at: %v", err) + } + if queuedArchived.Valid { + t.Fatalf("queued task archived_at = %s, want NULL", queuedArchived.Time) + } +} + +func assertPGArchivedCount( + t *testing.T, db *sql.DB, tenantID, rootAgentID uuid.UUID, want int, +) { + t.Helper() + var got int + if err := db.QueryRow( + `SELECT COUNT(*) FROM subagent_tasks + WHERE tenant_id = $1 AND root_agent_id = $2 AND archived_at IS NOT NULL`, + tenantID, rootAgentID, + ).Scan(&got); err != nil { + t.Fatalf("count archived tasks for %s/%s: %v", tenantID, rootAgentID, err) + } + if got != want { + t.Fatalf("archived tasks for %s/%s = %d, want %d", tenantID, rootAgentID, got, want) + } +} diff --git a/internal/store/sqlitestore/agent-links.go b/internal/store/sqlitestore/agent-links.go index 59ac0c1b..245f8739 100644 --- a/internal/store/sqlitestore/agent-links.go +++ b/internal/store/sqlitestore/agent-links.go @@ -6,6 +6,7 @@ import ( "context" "database/sql" "encoding/json" + "errors" "fmt" "strings" "time" @@ -207,6 +208,7 @@ func (s *SQLiteAgentLinkStore) GetLinkBetween(ctx context.Context, fromAgentID, tenantFilter = " AND tenant_id = ?" args = append(args, tenantID) } + args = append(args, fromAgentID) row := s.db.QueryRowContext(ctx, `SELECT `+linkSelectCols+` FROM agent_links WHERE status = 'active' @@ -214,11 +216,16 @@ func (s *SQLiteAgentLinkStore) GetLinkBetween(ctx context.Context, fromAgentID, (source_agent_id = ? AND target_agent_id = ? AND direction IN ('outbound', 'bidirectional')) OR (source_agent_id = ? AND target_agent_id = ? AND direction IN ('inbound', 'bidirectional')) - )`+tenantFilter+` LIMIT 1`, args...) + )`+tenantFilter+` + ORDER BY CASE WHEN source_agent_id = ? THEN 0 ELSE 1 END, created_at, id + LIMIT 1`, args...) d, err := scanLinkRow(row) - if err != nil { + if errors.Is(err, sql.ErrNoRows) { return nil, nil } + if err != nil { + return nil, err + } return d, nil } diff --git a/internal/store/sqlitestore/agent-links_test.go b/internal/store/sqlitestore/agent-links_test.go new file mode 100644 index 00000000..1b59cfe9 --- /dev/null +++ b/internal/store/sqlitestore/agent-links_test.go @@ -0,0 +1,224 @@ +//go:build sqlite || sqliteonly + +package sqlitestore + +import ( + "context" + "database/sql" + "testing" + + "github.com/google/uuid" + + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +func seedSQLiteLinkAgent(t *testing.T, db *sql.DB, tenantID uuid.UUID, key string) uuid.UUID { + t.Helper() + id := uuid.New() + _, err := db.Exec( + `INSERT INTO agents (id, tenant_id, agent_key, agent_type, status, provider, model, owner_id) + VALUES (?,?,?,'predefined','active','test','test-model','owner')`, + id.String(), tenantID.String(), key) + if err != nil { + t.Fatalf("seed agent %q: %v", key, err) + } + return id +} + +func newSQLiteLinkFixture(t *testing.T) (*SQLiteAgentLinkStore, *sql.DB, context.Context, uuid.UUID, uuid.UUID, uuid.UUID) { + t.Helper() + db := newHookTestDB(t) + tenantID, agentA := seedHookTenantAgent(t, db) + if _, err := db.Exec(`UPDATE agents SET agent_key = 'agent-a' WHERE id = ?`, agentA.String()); err != nil { + t.Fatalf("rename agent A: %v", err) + } + agentB := seedSQLiteLinkAgent(t, db, tenantID, "agent-b") + agentC := seedSQLiteLinkAgent(t, db, tenantID, "agent-c") + ctx := store.WithTenantID(context.Background(), tenantID) + return NewSQLiteAgentLinkStore(db), db, ctx, agentA, agentB, agentC +} + +func sqliteLink(source, target uuid.UUID, direction string) *store.AgentLinkData { + return &store.AgentLinkData{ + SourceAgentID: source, + TargetAgentID: target, + Direction: direction, + Status: store.LinkStatusActive, + MaxConcurrent: 1, + CreatedBy: "test", + } +} + +func targetKeys(t *testing.T, linkStore *SQLiteAgentLinkStore, ctx context.Context, from uuid.UUID) []string { + t.Helper() + targets, err := linkStore.DelegateTargets(ctx, from) + if err != nil { + t.Fatalf("DelegateTargets(%s): %v", from, err) + } + keys := make([]string, len(targets)) + for i := range targets { + keys[i] = targets[i].TargetAgentKey + } + return keys +} + +func TestSQLiteAgentLinkStoreDelegateTargetsDirections(t *testing.T) { + tests := []struct { + name string + direction string + wantFromA []string + wantFromB []string + }{ + { + name: "outbound", + direction: store.LinkDirectionOutbound, + wantFromA: []string{"agent-b"}, + }, + { + name: "inbound", + direction: store.LinkDirectionInbound, + wantFromB: []string{"agent-a"}, + }, + { + name: "bidirectional", + direction: store.LinkDirectionBidirectional, + wantFromA: []string{"agent-b"}, + wantFromB: []string{"agent-a"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + linkStore, _, ctx, agentA, agentB, _ := newSQLiteLinkFixture(t) + if err := linkStore.CreateLink(ctx, sqliteLink(agentA, agentB, tt.direction)); err != nil { + t.Fatalf("CreateLink: %v", err) + } + + if got := targetKeys(t, linkStore, ctx, agentA); !slicesEqual(got, tt.wantFromA) { + t.Fatalf("targets from A = %v, want %v", got, tt.wantFromA) + } + if got := targetKeys(t, linkStore, ctx, agentB); !slicesEqual(got, tt.wantFromB) { + t.Fatalf("targets from B = %v, want %v", got, tt.wantFromB) + } + }) + } +} + +func TestSQLiteAgentLinkStoreDelegateTargetsRefreshesAfterMutations(t *testing.T) { + linkStore, db, ctx, agentA, agentB, _ := newSQLiteLinkFixture(t) + link := sqliteLink(agentA, agentB, store.LinkDirectionOutbound) + if err := linkStore.CreateLink(ctx, link); err != nil { + t.Fatalf("CreateLink: %v", err) + } + if got := targetKeys(t, linkStore, ctx, agentA); !slicesEqual(got, []string{"agent-b"}) { + t.Fatalf("initial targets = %v", got) + } + + if err := linkStore.UpdateLink(ctx, link.ID, map[string]any{"status": store.LinkStatusDisabled}); err != nil { + t.Fatalf("disable link: %v", err) + } + if got := targetKeys(t, linkStore, ctx, agentA); len(got) != 0 { + t.Fatalf("targets after disable = %v, want none", got) + } + + if err := linkStore.UpdateLink(ctx, link.ID, map[string]any{ + "status": store.LinkStatusActive, + "direction": store.LinkDirectionInbound, + }); err != nil { + t.Fatalf("reverse link direction: %v", err) + } + if got := targetKeys(t, linkStore, ctx, agentA); len(got) != 0 { + t.Fatalf("targets from A after inbound update = %v, want none", got) + } + if got := targetKeys(t, linkStore, ctx, agentB); !slicesEqual(got, []string{"agent-a"}) { + t.Fatalf("targets from B after inbound update = %v", got) + } + + if _, err := db.Exec(`UPDATE agents SET status = 'inactive' WHERE id = ?`, agentA.String()); err != nil { + t.Fatalf("deactivate effective target: %v", err) + } + if got := targetKeys(t, linkStore, ctx, agentB); len(got) != 0 { + t.Fatalf("targets with inactive effective target = %v, want none", got) + } + + if _, err := db.Exec(`UPDATE agents SET status = 'active' WHERE id = ?`, agentA.String()); err != nil { + t.Fatalf("reactivate target: %v", err) + } + if err := linkStore.DeleteLink(ctx, link.ID); err != nil { + t.Fatalf("DeleteLink: %v", err) + } + if got := targetKeys(t, linkStore, ctx, agentB); len(got) != 0 { + t.Fatalf("targets after delete = %v, want none", got) + } +} + +func TestSQLiteAgentLinkStoreDelegateTargetsTenantScoped(t *testing.T) { + linkStore, _, ctx, agentA, agentB, _ := newSQLiteLinkFixture(t) + if err := linkStore.CreateLink(ctx, sqliteLink(agentA, agentB, store.LinkDirectionOutbound)); err != nil { + t.Fatalf("CreateLink: %v", err) + } + + otherTenantCtx := store.WithTenantID(context.Background(), uuid.New()) + if got := targetKeys(t, linkStore, otherTenantCtx, agentA); len(got) != 0 { + t.Fatalf("cross-tenant targets = %v, want none", got) + } +} + +func TestSQLiteAgentLinkStoreGetLinkBetweenNotFound(t *testing.T) { + linkStore, _, ctx, agentA, agentB, _ := newSQLiteLinkFixture(t) + + link, err := linkStore.GetLinkBetween(ctx, agentA, agentB) + if err != nil { + t.Fatalf("GetLinkBetween: %v", err) + } + if link != nil { + t.Fatalf("GetLinkBetween = %#v, want nil", link) + } +} + +func TestSQLiteAgentLinkStoreGetLinkBetweenPrefersDelegatorAuthoredRow(t *testing.T) { + linkStore, _, ctx, agentA, agentB, _ := newSQLiteLinkFixture(t) + + reverse := sqliteLink(agentB, agentA, store.LinkDirectionInbound) + if err := linkStore.CreateLink(ctx, reverse); err != nil { + t.Fatalf("CreateLink(reverse): %v", err) + } + direct := sqliteLink(agentA, agentB, store.LinkDirectionOutbound) + if err := linkStore.CreateLink(ctx, direct); err != nil { + t.Fatalf("CreateLink(direct): %v", err) + } + + link, err := linkStore.GetLinkBetween(ctx, agentA, agentB) + if err != nil { + t.Fatalf("GetLinkBetween: %v", err) + } + if link == nil { + t.Fatal("GetLinkBetween returned nil") + } + if link.ID != direct.ID { + t.Fatalf("GetLinkBetween chose %s, want delegator-authored %s", link.ID, direct.ID) + } +} + +func TestSQLiteAgentLinkStoreGetLinkBetweenPropagatesDatabaseError(t *testing.T) { + linkStore, db, ctx, agentA, agentB, _ := newSQLiteLinkFixture(t) + if err := db.Close(); err != nil { + t.Fatalf("close fixture DB: %v", err) + } + + if _, err := linkStore.GetLinkBetween(ctx, agentA, agentB); err == nil { + t.Fatal("GetLinkBetween error = nil after database close") + } +} + +func slicesEqual(got, want []string) bool { + if len(got) != len(want) { + return false + } + for i := range got { + if got[i] != want[i] { + return false + } + } + return true +} diff --git a/internal/store/sqlitestore/factory.go b/internal/store/sqlitestore/factory.go index c679e002..1371dcef 100644 --- a/internal/store/sqlitestore/factory.go +++ b/internal/store/sqlitestore/factory.go @@ -34,6 +34,7 @@ func NewSQLiteStores(cfg store.StoreConfig) (*store.Stores, error) { } else { slog.Warn("securecli: encryption key empty, store disabled") } + subagentTasks := NewSQLiteSubagentTaskStore(db) sqliteStores := &store.Stores{ DB: db, @@ -67,7 +68,8 @@ func NewSQLiteStores(cfg store.StoreConfig) (*store.Stores, error) { ConfigPermissions: NewSQLiteConfigPermissionStore(db), BrowserCookies: NewSQLiteBrowserCookieStore(db, cfg.EncryptionKey), Memory: NewSQLiteMemoryStore(db), - SubagentTasks: NewSQLiteSubagentTaskStore(db), + SubagentTasks: subagentTasks, + SubagentTaskRecovery: subagentTasks, AgentLinks: NewSQLiteAgentLinkStore(db), SecureCLI: secureCLI, SecureCLIGrants: NewSQLiteSecureCLIAgentGrantStore(db, cfg.EncryptionKey), diff --git a/internal/store/sqlitestore/schema.go b/internal/store/sqlitestore/schema.go index bebb6dcd..ef946e56 100644 --- a/internal/store/sqlitestore/schema.go +++ b/internal/store/sqlitestore/schema.go @@ -16,7 +16,7 @@ var schemaSQL string // SchemaVersion is the current SQLite schema version. // Bump this when adding new migration steps below. -const SchemaVersion = 58 +const SchemaVersion = 59 // migrations maps version → SQL to apply when upgrading FROM that version. // schema.sql always represents the LATEST full schema (for fresh DBs). @@ -29,7 +29,78 @@ const SchemaVersion = 58 // } // // Then bump SchemaVersion to 2. +const sqliteSubagentRootAgentScopeMigrationBody = `UPDATE subagent_tasks +SET root_agent_id = json_extract(metadata, '$.root_agent_id') +WHERE root_agent_id IS NULL + AND json_valid(metadata) + AND typeof(json_extract(metadata, '$.root_agent_id')) = 'text' + AND EXISTS ( + SELECT 1 + FROM agents + WHERE agents.tenant_id = subagent_tasks.tenant_id + AND agents.id = json_extract(subagent_tasks.metadata, '$.root_agent_id') + ); +UPDATE subagent_tasks +SET root_agent_id = ( + SELECT MIN(agents.id) + FROM agents + WHERE agents.tenant_id = subagent_tasks.tenant_id + AND agents.agent_key = subagent_tasks.parent_agent_key + AND agents.created_at < subagent_tasks.created_at +) +WHERE root_agent_id IS NULL + AND CASE + WHEN json_valid(metadata) THEN json_type(metadata, '$.root_agent_id') IS NULL + ELSE 1 + END + AND ( + SELECT COUNT(*) + FROM agents + WHERE agents.tenant_id = subagent_tasks.tenant_id + AND agents.agent_key = subagent_tasks.parent_agent_key + AND agents.created_at < subagent_tasks.created_at + ) = 1; +CREATE INDEX IF NOT EXISTS idx_subagent_tasks_root_status + ON subagent_tasks(tenant_id, root_agent_id, status, created_at DESC) + WHERE root_agent_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_subagent_tasks_root_session + ON subagent_tasks(tenant_id, root_agent_id, session_key, created_at DESC) + WHERE root_agent_id IS NOT NULL AND session_key IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_subagent_tasks_root_archive + ON subagent_tasks(tenant_id, root_agent_id, completed_at, id) + WHERE root_agent_id IS NOT NULL + AND status IN ('completed', 'failed', 'cancelled') + AND archived_at IS NULL; +CREATE TRIGGER IF NOT EXISTS trg_subagent_tasks_root_tenant_insert +BEFORE INSERT ON subagent_tasks +WHEN NEW.root_agent_id IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM agents + WHERE agents.id = NEW.root_agent_id + AND agents.tenant_id = NEW.tenant_id + ) +BEGIN + SELECT RAISE(ABORT, 'subagent root agent belongs to another tenant'); +END; +CREATE TRIGGER IF NOT EXISTS trg_subagent_tasks_root_tenant_update +BEFORE UPDATE OF root_agent_id, tenant_id ON subagent_tasks +WHEN NEW.root_agent_id IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM agents + WHERE agents.id = NEW.root_agent_id + AND agents.tenant_id = NEW.tenant_id + ) +BEGIN + SELECT RAISE(ABORT, 'subagent root agent belongs to another tenant'); +END;` + var migrations = map[int]string{ + // Version 58 → 59: scope persisted subagent tasks by immutable root-agent UUID. + // Metadata is authoritative; key fallback is allowed only for one matching + // agent that predates the task. Unmatched rows remain inaccessible. + 58: `ALTER TABLE subagent_tasks + ADD COLUMN root_agent_id TEXT REFERENCES agents(id) ON DELETE SET NULL; +` + sqliteSubagentRootAgentScopeMigrationBody, // Version 57 → 58: restore custom skills previously converted by the bundled skill seeder. 57: `UPDATE skills SET is_system = 0, @@ -1521,6 +1592,12 @@ func EnsureSchema(db *sql.DB) error { return fmt.Errorf("inspect channel pending message parent column: %w", err) } } + if v == 58 { + patch, err = sqliteSubagentRootAgentMigrationPatch(db) + if err != nil { + return fmt.Errorf("inspect subagent task root-agent column: %w", err) + } + } // Migrations that rebuild a table referenced by another table's FK // require foreign_keys=OFF per SQLite altertable §7. The pragma is // a no-op inside a transaction, so toggle it around BEGIN/COMMIT. @@ -1681,6 +1758,19 @@ func sqlitePendingMessageParentMigrationPatch(db *sql.DB) (string, error) { return patch, nil } +func sqliteSubagentRootAgentMigrationPatch(db *sql.DB) (string, error) { + hasColumn, err := sqliteColumnExists(db, "subagent_tasks", "root_agent_id") + if err != nil { + return "", err + } + patch := "" + if !hasColumn { + patch += "ALTER TABLE subagent_tasks ADD COLUMN root_agent_id TEXT REFERENCES agents(id) ON DELETE SET NULL;\n" + } + patch += sqliteSubagentRootAgentScopeMigrationBody + return patch, nil +} + func sqliteColumnExists(db *sql.DB, tableName, columnName string) (bool, error) { rows, err := db.Query("PRAGMA table_info(" + tableName + ")") if err != nil { diff --git a/internal/store/sqlitestore/schema.sql b/internal/store/sqlitestore/schema.sql index c6856b46..92539dee 100644 --- a/internal/store/sqlitestore/schema.sql +++ b/internal/store/sqlitestore/schema.sql @@ -1678,6 +1678,7 @@ CREATE INDEX IF NOT EXISTS idx_system_configs_tenant ON system_configs(tenant_id CREATE TABLE IF NOT EXISTS subagent_tasks ( id TEXT PRIMARY KEY, tenant_id TEXT NOT NULL REFERENCES tenants(id) ON DELETE CASCADE, + root_agent_id TEXT REFERENCES agents(id) ON DELETE SET NULL, parent_agent_key VARCHAR(255) NOT NULL, session_key VARCHAR(500), subject VARCHAR(255) NOT NULL, @@ -1706,6 +1707,39 @@ CREATE TABLE IF NOT EXISTS subagent_tasks ( CREATE INDEX IF NOT EXISTS idx_subagent_tasks_parent_status ON subagent_tasks(tenant_id, parent_agent_key, status); CREATE INDEX IF NOT EXISTS idx_subagent_tasks_session ON subagent_tasks(session_key); CREATE INDEX IF NOT EXISTS idx_subagent_tasks_created ON subagent_tasks(tenant_id, created_at); +CREATE INDEX IF NOT EXISTS idx_subagent_tasks_root_status + ON subagent_tasks(tenant_id, root_agent_id, status, created_at DESC) + WHERE root_agent_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_subagent_tasks_root_session + ON subagent_tasks(tenant_id, root_agent_id, session_key, created_at DESC) + WHERE root_agent_id IS NOT NULL AND session_key IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_subagent_tasks_root_archive + ON subagent_tasks(tenant_id, root_agent_id, completed_at, id) + WHERE root_agent_id IS NOT NULL + AND status IN ('completed', 'failed', 'cancelled') + AND archived_at IS NULL; +CREATE TRIGGER IF NOT EXISTS trg_subagent_tasks_root_tenant_insert +BEFORE INSERT ON subagent_tasks +WHEN NEW.root_agent_id IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM agents + WHERE agents.id = NEW.root_agent_id + AND agents.tenant_id = NEW.tenant_id + ) +BEGIN + SELECT RAISE(ABORT, 'subagent root agent belongs to another tenant'); +END; +CREATE TRIGGER IF NOT EXISTS trg_subagent_tasks_root_tenant_update +BEFORE UPDATE OF root_agent_id, tenant_id ON subagent_tasks +WHEN NEW.root_agent_id IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM agents + WHERE agents.id = NEW.root_agent_id + AND agents.tenant_id = NEW.tenant_id + ) +BEGIN + SELECT RAISE(ABORT, 'subagent root agent belongs to another tenant'); +END; -- ============================================================ -- Table: episodic_summaries (V3 Tier 2 memory) diff --git a/internal/store/sqlitestore/subagent-tasks.go b/internal/store/sqlitestore/subagent-tasks.go index b09b1025..08617780 100644 --- a/internal/store/sqlitestore/subagent-tasks.go +++ b/internal/store/sqlitestore/subagent-tasks.go @@ -22,16 +22,18 @@ type SQLiteSubagentTaskStore struct { db *sql.DB } +var _ store.SubagentTaskRecoveryStore = (*SQLiteSubagentTaskStore)(nil) + // NewSQLiteSubagentTaskStore creates a new SQLiteSubagentTaskStore. func NewSQLiteSubagentTaskStore(db *sql.DB) *SQLiteSubagentTaskStore { return &SQLiteSubagentTaskStore{db: db} } -const subagentTaskInsertCols = `tenant_id, parent_agent_key, session_key, subject, description, +const subagentTaskInsertCols = `tenant_id, root_agent_id, parent_agent_key, session_key, subject, description, status, result, depth, model, provider, iterations, input_tokens, output_tokens, origin_channel, origin_chat_id, origin_peer_kind, origin_user_id, spawned_by, metadata` -const subagentTaskSelectCols = `id, tenant_id, parent_agent_key, session_key, subject, description, +const subagentTaskSelectCols = `id, tenant_id, root_agent_id, parent_agent_key, session_key, subject, description, status, result, depth, model, provider, iterations, input_tokens, output_tokens, origin_channel, origin_chat_id, origin_peer_kind, origin_user_id, spawned_by, completed_at, archived_at, COALESCE(metadata, '{}'), created_at, updated_at` @@ -39,6 +41,9 @@ const subagentTaskSelectCols = `id, tenant_id, parent_agent_key, session_key, su // Create persists a new subagent task at spawn time. func (s *SQLiteSubagentTaskStore) Create(ctx context.Context, task *store.SubagentTaskData) error { tid := tenantIDForInsert(ctx) + if task.RootAgentID == uuid.Nil { + return store.ErrSubagentRootAgentIDRequired + } metaJSON := []byte("{}") if len(task.Metadata) > 0 { @@ -49,10 +54,10 @@ func (s *SQLiteSubagentTaskStore) Create(ctx context.Context, task *store.Subage now := time.Now().UTC().Format(time.RFC3339Nano) q := fmt.Sprintf(`INSERT OR IGNORE INTO subagent_tasks (id, %s, created_at, updated_at) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, subagentTaskInsertCols) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, subagentTaskInsertCols) _, err := s.db.ExecContext(ctx, q, - task.ID, tid, task.ParentAgentKey, task.SessionKey, task.Subject, task.Description, + task.ID, tid, task.RootAgentID, task.ParentAgentKey, task.SessionKey, task.Subject, task.Description, task.Status, task.Result, task.Depth, task.Model, task.Provider, task.Iterations, task.InputTokens, task.OutputTokens, task.OriginChannel, task.OriginChatID, task.OriginPeerKind, task.OriginUserID, @@ -70,7 +75,7 @@ func scanTask(row interface{ Scan(...any) error }) (*store.SubagentTaskData, err var createdAt, updatedAt sqliteTime err := row.Scan( - &t.ID, &t.TenantID, &t.ParentAgentKey, &t.SessionKey, &t.Subject, &t.Description, + &t.ID, &t.TenantID, &t.RootAgentID, &t.ParentAgentKey, &t.SessionKey, &t.Subject, &t.Description, &t.Status, &t.Result, &t.Depth, &t.Model, &t.Provider, &t.Iterations, &t.InputTokens, &t.OutputTokens, &t.OriginChannel, &t.OriginChatID, &t.OriginPeerKind, &t.OriginUserID, &t.SpawnedBy, @@ -95,14 +100,20 @@ func scanTask(row interface{ Scan(...any) error }) (*store.SubagentTaskData, err return &t, nil } -// Get retrieves a single task by ID (tenant-scoped). -func (s *SQLiteSubagentTaskStore) Get(ctx context.Context, id uuid.UUID) (*store.SubagentTaskData, error) { +// Get retrieves a task owned by the tenant and immutable root-agent UUID. +func (s *SQLiteSubagentTaskStore) Get( + ctx context.Context, rootAgentID, id uuid.UUID, +) (*store.SubagentTaskData, error) { tid, err := requireTenantID(ctx) if err != nil { return nil, err } - q := fmt.Sprintf(`SELECT %s FROM subagent_tasks WHERE id = ? AND tenant_id = ?`, subagentTaskSelectCols) - row := s.db.QueryRowContext(ctx, q, id, tid) + if rootAgentID == uuid.Nil { + return nil, store.ErrSubagentRootAgentIDRequired + } + q := fmt.Sprintf(`SELECT %s FROM subagent_tasks + WHERE id = ? AND tenant_id = ? AND root_agent_id = ?`, subagentTaskSelectCols) + row := s.db.QueryRowContext(ctx, q, id, tid, rootAgentID) t, err := scanTask(row) if errors.Is(err, sql.ErrNoRows) { return nil, nil @@ -112,7 +123,7 @@ func (s *SQLiteSubagentTaskStore) Get(ctx context.Context, id uuid.UUID) (*store // UpdateStatus updates status, result, iterations, and token counts on completion/failure. func (s *SQLiteSubagentTaskStore) UpdateStatus( - ctx context.Context, id uuid.UUID, + ctx context.Context, rootAgentID, id uuid.UUID, status string, result *string, iterations int, inputTokens, outputTokens int64, ) error { @@ -120,10 +131,13 @@ func (s *SQLiteSubagentTaskStore) UpdateStatus( if err != nil { return err } + if rootAgentID == uuid.Nil { + return store.ErrSubagentRootAgentIDRequired + } now := time.Now().UTC().Format(time.RFC3339Nano) var completedAt *string - if status != "running" { + if store.IsTerminalSubagentTaskStatus(status) { v := now completedAt = &v } @@ -132,34 +146,49 @@ func (s *SQLiteSubagentTaskStore) UpdateStatus( status = ?, result = ?, iterations = ?, input_tokens = ?, output_tokens = ?, completed_at = ?, updated_at = ? - WHERE id = ? AND tenant_id = ?` - _, err = s.db.ExecContext(ctx, q, + WHERE id = ? AND tenant_id = ? AND root_agent_id = ?` + res, err := s.db.ExecContext(ctx, q, status, result, iterations, inputTokens, outputTokens, - completedAt, now, id, tid, + completedAt, now, id, tid, rootAgentID, ) - return err + if err != nil { + return err + } + affected, err := res.RowsAffected() + if err != nil { + return err + } + if affected != 1 { + return store.ErrSubagentTaskNotFound + } + return nil } -// ListByParent returns tasks for a parent agent key, optionally filtered by status. +// ListByParent returns tasks for a root-agent UUID, optionally filtered by status. func (s *SQLiteSubagentTaskStore) ListByParent( - ctx context.Context, parentAgentKey string, statusFilter string, + ctx context.Context, rootAgentID uuid.UUID, statusFilter string, ) ([]store.SubagentTaskData, error) { tid, err := requireTenantID(ctx) if err != nil { return nil, err } + if rootAgentID == uuid.Nil { + return nil, store.ErrSubagentRootAgentIDRequired + } var rows *sql.Rows if statusFilter != "" { q := fmt.Sprintf(`SELECT %s FROM subagent_tasks - WHERE tenant_id = ? AND parent_agent_key = ? AND status = ? - ORDER BY created_at DESC LIMIT 50`, subagentTaskSelectCols) - rows, err = s.db.QueryContext(ctx, q, tid, parentAgentKey, statusFilter) + WHERE tenant_id = ? AND root_agent_id = ? AND status = ? + AND COALESCE(json_extract(metadata, '$.completion_kind'), 'subagent') <> 'delegate' + ORDER BY created_at DESC LIMIT 50`, subagentTaskSelectCols) + rows, err = s.db.QueryContext(ctx, q, tid, rootAgentID, statusFilter) } else { q := fmt.Sprintf(`SELECT %s FROM subagent_tasks - WHERE tenant_id = ? AND parent_agent_key = ? - ORDER BY created_at DESC LIMIT 50`, subagentTaskSelectCols) - rows, err = s.db.QueryContext(ctx, q, tid, parentAgentKey) + WHERE tenant_id = ? AND root_agent_id = ? + AND COALESCE(json_extract(metadata, '$.completion_kind'), 'subagent') <> 'delegate' + ORDER BY created_at DESC LIMIT 50`, subagentTaskSelectCols) + rows, err = s.db.QueryContext(ctx, q, tid, rootAgentID) } if err != nil { return nil, err @@ -170,17 +199,21 @@ func (s *SQLiteSubagentTaskStore) ListByParent( // ListBySession returns tasks for a specific session key (tenant-scoped). func (s *SQLiteSubagentTaskStore) ListBySession( - ctx context.Context, sessionKey string, + ctx context.Context, rootAgentID uuid.UUID, sessionKey string, ) ([]store.SubagentTaskData, error) { tid, err := requireTenantID(ctx) if err != nil { return nil, err } + if rootAgentID == uuid.Nil { + return nil, store.ErrSubagentRootAgentIDRequired + } q := fmt.Sprintf(`SELECT %s FROM subagent_tasks - WHERE tenant_id = ? AND session_key = ? + WHERE tenant_id = ? AND root_agent_id = ? AND session_key = ? + AND COALESCE(json_extract(metadata, '$.completion_kind'), 'subagent') <> 'delegate' ORDER BY created_at DESC LIMIT 50`, subagentTaskSelectCols) - rows, err := s.db.QueryContext(ctx, q, tid, sessionKey) + rows, err := s.db.QueryContext(ctx, q, tid, rootAgentID, sessionKey) if err != nil { return nil, err } @@ -188,14 +221,38 @@ func (s *SQLiteSubagentTaskStore) ListBySession( return collectTasks(rows) } -// Archive marks old completed/failed/cancelled tasks as archived. -func (s *SQLiteSubagentTaskStore) Archive(ctx context.Context, olderThan time.Duration) (int64, error) { +// Archive marks a bounded batch of old terminal tasks as archived. +func (s *SQLiteSubagentTaskStore) Archive( + ctx context.Context, rootAgentID uuid.UUID, olderThan time.Duration, limit int, +) (int64, error) { + tid, err := requireTenantID(ctx) + if err != nil { + return 0, err + } + if rootAgentID == uuid.Nil { + return 0, store.ErrSubagentRootAgentIDRequired + } + if limit <= 0 { + return 0, nil + } + cutoff := time.Now().UTC().Add(-olderThan).Format(time.RFC3339Nano) now := time.Now().UTC().Format(time.RFC3339Nano) q := `UPDATE subagent_tasks SET archived_at = ?, updated_at = ? - WHERE status IN ('completed', 'failed', 'cancelled') - AND archived_at IS NULL AND completed_at < ?` - res, err := s.db.ExecContext(ctx, q, now, now, cutoff) + WHERE tenant_id = ? AND root_agent_id = ? AND id IN ( + SELECT id + FROM subagent_tasks + WHERE tenant_id = ? AND root_agent_id = ? + AND status IN ('completed', 'failed', 'cancelled') + AND archived_at IS NULL AND completed_at < ? + ORDER BY completed_at, id + LIMIT ? + )` + res, err := s.db.ExecContext( + ctx, q, + now, now, tid, rootAgentID, + tid, rootAgentID, cutoff, limit, + ) if err != nil { return 0, err } @@ -204,11 +261,16 @@ func (s *SQLiteSubagentTaskStore) Archive(ctx context.Context, olderThan time.Du // UpdateMetadata merges metadata keys atomically using json_set(). // Builds a single UPDATE statement to avoid read-merge-write race window. -func (s *SQLiteSubagentTaskStore) UpdateMetadata(ctx context.Context, id uuid.UUID, metadata map[string]any) error { +func (s *SQLiteSubagentTaskStore) UpdateMetadata( + ctx context.Context, rootAgentID, id uuid.UUID, metadata map[string]any, +) error { tid, err := requireTenantID(ctx) if err != nil { return err } + if rootAgentID == uuid.Nil { + return store.ErrSubagentRootAgentIDRequired + } if len(metadata) == 0 { return nil } @@ -221,18 +283,46 @@ func (s *SQLiteSubagentTaskStore) UpdateMetadata(ctx context.Context, id uuid.UU if !validMetadataKey(k) { return fmt.Errorf("invalid metadata key: %q", k) } - parts = append(parts, fmt.Sprintf("'$.%s', ?", k)) + parts = append(parts, fmt.Sprintf("'$.%s', json(?)", k)) b, _ := json.Marshal(v) args = append(args, string(b)) } now := time.Now().UTC().Format(time.RFC3339Nano) setExpr := "json_set(metadata, " + strings.Join(parts, ", ") + ")" - args = append(args, now, id, tid) + args = append(args, now, id, tid, rootAgentID) - q := fmt.Sprintf(`UPDATE subagent_tasks SET metadata = %s, updated_at = ? WHERE id = ? AND tenant_id = ?`, setExpr) - _, err = s.db.ExecContext(ctx, q, args...) - return err + q := fmt.Sprintf(`UPDATE subagent_tasks SET metadata = %s, updated_at = ? + WHERE id = ? AND tenant_id = ? AND root_agent_id = ?`, setExpr) + res, err := s.db.ExecContext(ctx, q, args...) + if err != nil { + return err + } + affected, err := res.RowsAffected() + if err != nil { + return err + } + if affected != 1 { + return store.ErrSubagentTaskNotFound + } + return nil +} + +// RecoverInterrupted marks child runs that cannot survive a process restart as +// failed. This cross-tenant update runs before the gateway accepts new work. +func (s *SQLiteSubagentTaskStore) RecoverInterrupted(ctx context.Context) (int64, error) { + if !store.IsMasterScope(ctx) { + return 0, fmt.Errorf("recover interrupted subagent tasks requires master scope") + } + now := time.Now().UTC().Format(time.RFC3339Nano) + q := `UPDATE subagent_tasks SET + status = 'failed', result = ?, completed_at = ?, updated_at = ? + WHERE status NOT IN ('completed', 'failed', 'cancelled')` + res, err := s.db.ExecContext(ctx, q, store.InterruptedSubagentTaskResult, now, now) + if err != nil { + return 0, err + } + return res.RowsAffected() } func collectTasks(rows *sql.Rows) ([]store.SubagentTaskData, error) { diff --git a/internal/store/sqlitestore/subagent-tasks_test.go b/internal/store/sqlitestore/subagent-tasks_test.go new file mode 100644 index 00000000..cec9a193 --- /dev/null +++ b/internal/store/sqlitestore/subagent-tasks_test.go @@ -0,0 +1,586 @@ +//go:build sqlite || sqliteonly + +package sqlitestore + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/google/uuid" + + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +func createSQLiteSubagentTask( + t *testing.T, + taskStore *SQLiteSubagentTaskStore, + ctx context.Context, + rootAgentID uuid.UUID, + rootAgentKey, sessionKey, status string, +) uuid.UUID { + t.Helper() + + id := uuid.Must(uuid.NewV7()) + task := &store.SubagentTaskData{ + RootAgentID: rootAgentID, + ParentAgentKey: rootAgentKey, + SessionKey: &sessionKey, + Subject: "store test", + Description: "verify scoped persistence", + Status: status, + Depth: 1, + Metadata: map[string]any{}, + } + task.ID = id + if err := taskStore.Create(ctx, task); err != nil { + t.Fatalf("Create(%s): %v", id, err) + } + return id +} + +func TestSQLiteSubagentTaskStoreRequiresTenantAndRootScope(t *testing.T) { + db := newHookTestDB(t) + tenantA, rootAID := seedHookTenantAgent(t, db) + tenantB, tenantBRootID := seedHookTenantAgent(t, db) + ctxA := sqliteTenantCtx(tenantA) + ctxB := sqliteTenantCtx(tenantB) + taskStore := NewSQLiteSubagentTaskStore(db) + + const ( + rootA = "root-a" + rootB = "root-b" + sessionID = "shared-session" + ) + rootBID := uuid.Must(uuid.NewV7()) + if _, err := db.Exec( + `INSERT INTO agents (id, tenant_id, agent_key, agent_type, status, provider, model, owner_id) + VALUES (?,?,?,'predefined','active','test','test-model','owner')`, + rootBID, tenantA, rootB, + ); err != nil { + t.Fatalf("seed second root agent: %v", err) + } + taskA := createSQLiteSubagentTask(t, taskStore, ctxA, rootAID, rootA, sessionID, "queued") + taskB := createSQLiteSubagentTask(t, taskStore, ctxA, rootBID, rootB, sessionID, "queued") + delegationTask := createSQLiteSubagentTask(t, taskStore, ctxA, rootAID, rootA, sessionID, "queued") + if err := taskStore.UpdateMetadata(ctxA, rootAID, delegationTask, map[string]any{ + "completion_kind": "delegate", + }); err != nil { + t.Fatalf("mark delegation completion: %v", err) + } + _ = createSQLiteSubagentTask(t, taskStore, ctxB, tenantBRootID, rootA, sessionID, "queued") + crossTenantTask := &store.SubagentTaskData{ + BaseModel: store.BaseModel{ID: uuid.Must(uuid.NewV7())}, + RootAgentID: tenantBRootID, + ParentAgentKey: rootA, + Subject: "cross tenant", + Description: "must be rejected", + Status: "queued", + Depth: 1, + } + if err := taskStore.Create(ctxA, crossTenantTask); err == nil { + t.Fatal("Create accepted root agent from another tenant") + } + + got, err := taskStore.Get(ctxA, rootAID, taskA) + if err != nil { + t.Fatalf("Get owning scope: %v", err) + } + if got == nil || got.ID != taskA { + t.Fatalf("Get owning scope = %#v, want task %s", got, taskA) + } + + got, err = taskStore.Get(ctxA, rootBID, taskA) + if err != nil { + t.Fatalf("Get cross-root: %v", err) + } + if got != nil { + t.Fatalf("Get cross-root = %#v, want nil", got) + } + + got, err = taskStore.Get(ctxB, rootAID, taskA) + if err != nil { + t.Fatalf("Get cross-tenant: %v", err) + } + if got != nil { + t.Fatalf("Get cross-tenant = %#v, want nil", got) + } + + if err := taskStore.UpdateStatus(ctxA, rootBID, taskA, "completed", nil, 3, 10, 20); !errors.Is(err, store.ErrSubagentTaskNotFound) { + t.Fatalf("UpdateStatus cross-root error = %v, want scoped not found", err) + } + got, err = taskStore.Get(ctxA, rootAID, taskA) + if err != nil { + t.Fatalf("Get after cross-root status update: %v", err) + } + if got.Status != "queued" || got.CompletedAt != nil { + t.Fatalf("cross-root status update changed task: status=%q completed_at=%v", got.Status, got.CompletedAt) + } + + if err := taskStore.UpdateMetadata(ctxA, rootBID, taskA, map[string]any{"denied": true}); !errors.Is(err, store.ErrSubagentTaskNotFound) { + t.Fatalf("UpdateMetadata cross-root error = %v, want scoped not found", err) + } + got, err = taskStore.Get(ctxA, rootAID, taskA) + if err != nil { + t.Fatalf("Get after cross-root metadata update: %v", err) + } + if _, exists := got.Metadata["denied"]; exists { + t.Fatalf("cross-root metadata update changed task: %#v", got.Metadata) + } + if err := taskStore.UpdateMetadata(ctxA, rootAID, taskA, map[string]any{ + "announcement_status": "undelivered", + "delivered": false, + }); err != nil { + t.Fatalf("UpdateMetadata owning scope: %v", err) + } + got, err = taskStore.Get(ctxA, rootAID, taskA) + if err != nil { + t.Fatalf("Get after owning metadata update: %v", err) + } + if got.Metadata["announcement_status"] != "undelivered" || got.Metadata["delivered"] != false { + t.Fatalf("metadata JSON types were not preserved: %#v", got.Metadata) + } + + parentTasksA, err := taskStore.ListByParent(ctxA, rootAID, "") + if err != nil { + t.Fatalf("ListByParent root A: %v", err) + } + if len(parentTasksA) != 1 || parentTasksA[0].ID != taskA { + t.Fatalf("ListByParent root A = %#v, want only self-clone %s", parentTasksA, taskA) + } + parentTasksB, err := taskStore.ListByParent(ctxA, rootBID, "queued") + if err != nil { + t.Fatalf("ListByParent root B: %v", err) + } + if len(parentTasksB) != 1 || parentTasksB[0].ID != taskB { + t.Fatalf("ListByParent root B = %#v, want only %s", parentTasksB, taskB) + } + + tasksA, err := taskStore.ListBySession(ctxA, rootAID, sessionID) + if err != nil { + t.Fatalf("ListBySession root A: %v", err) + } + if len(tasksA) != 1 || tasksA[0].ID != taskA { + t.Fatalf("ListBySession root A = %#v, want only self-clone %s", tasksA, taskA) + } + tasksB, err := taskStore.ListBySession(ctxA, rootBID, sessionID) + if err != nil { + t.Fatalf("ListBySession root B: %v", err) + } + if len(tasksB) != 1 || tasksB[0].ID != taskB { + t.Fatalf("ListBySession root B = %#v, want only %s", tasksB, taskB) + } + + if _, err := taskStore.Get(context.Background(), rootAID, taskA); err == nil { + t.Fatal("Get without tenant context returned nil error") + } + if _, err := taskStore.Get(ctxA, uuid.Nil, taskA); !errors.Is(err, store.ErrSubagentRootAgentIDRequired) { + t.Fatalf("Get empty root error = %v, want %v", err, store.ErrSubagentRootAgentIDRequired) + } +} + +func TestSQLiteSubagentTaskStoreRejectsRecreatedAgentWithSameKey(t *testing.T) { + db := newHookTestDB(t) + tenantID, oldRootAgentID := seedHookTenantAgent(t, db) + ctx := sqliteTenantCtx(tenantID) + taskStore := NewSQLiteSubagentTaskStore(db) + + const rootAgentKey = "recreated-root" + if _, err := db.Exec( + `UPDATE agents SET agent_key = ? WHERE id = ?`, + rootAgentKey, oldRootAgentID, + ); err != nil { + t.Fatalf("rename original root agent: %v", err) + } + taskID := createSQLiteSubagentTask( + t, taskStore, ctx, oldRootAgentID, rootAgentKey, "recreated-root-session", "queued", + ) + + if _, err := db.Exec(`DELETE FROM agents WHERE id = ?`, oldRootAgentID); err != nil { + t.Fatalf("delete original root agent: %v", err) + } + newRootAgentID := uuid.Must(uuid.NewV7()) + if _, err := db.Exec( + `INSERT INTO agents (id, tenant_id, agent_key, agent_type, status, provider, model, owner_id) + VALUES (?,?,?,'predefined','active','test','test-model','owner')`, + newRootAgentID, tenantID, rootAgentKey, + ); err != nil { + t.Fatalf("recreate root agent: %v", err) + } + + got, err := taskStore.Get(ctx, newRootAgentID, taskID) + if err != nil { + t.Fatalf("Get with recreated root agent: %v", err) + } + if got != nil { + t.Fatalf("Get with recreated root agent returned old task %s", got.ID) + } + if err := taskStore.UpdateStatus(ctx, newRootAgentID, taskID, "completed", nil, 1, 2, 3); !errors.Is(err, store.ErrSubagentTaskNotFound) { + t.Fatalf("UpdateStatus with recreated root agent error = %v, want scoped not found", err) + } + var status string + var rootAgentID sql.NullString + if err := db.QueryRow( + `SELECT status, root_agent_id FROM subagent_tasks WHERE id = ?`, taskID, + ).Scan(&status, &rootAgentID); err != nil { + t.Fatalf("read preserved old task: %v", err) + } + if status != "queued" || rootAgentID.Valid { + t.Fatalf("old task after recreate: status=%q root_agent_id=%q, want queued/NULL", status, rootAgentID.String) + } +} + +func TestSQLiteSubagentTaskMigrationBackfillsOnlySafeOwners(t *testing.T) { + db := newHookTestDB(t) + tenantID, oldRootAgentID := seedHookTenantAgent(t, db) + + const rootAgentKey = "migration-recreated-root" + if _, err := db.Exec( + `UPDATE agents SET agent_key = ?, deleted_at = ? WHERE id = ?`, + rootAgentKey, time.Now().UTC().Format(time.RFC3339Nano), oldRootAgentID, + ); err != nil { + t.Fatalf("prepare historical root agent: %v", err) + } + newRootAgentID := uuid.Must(uuid.NewV7()) + if _, err := db.Exec( + `INSERT INTO agents (id, tenant_id, agent_key, agent_type, status, provider, model, owner_id) + VALUES (?,?,?,'predefined','active','test','test-model','owner')`, + newRootAgentID, tenantID, rootAgentKey, + ); err != nil { + t.Fatalf("insert recreated root agent: %v", err) + } + + const ( + safeRootKey = "migration-safe-root" + recreatedOnlyKey = "migration-hard-deleted-root" + ) + safeRootAgentID := uuid.Must(uuid.NewV7()) + recreatedOnlyAgentID := uuid.Must(uuid.NewV7()) + if _, err := db.Exec( + `INSERT INTO agents + (id, tenant_id, agent_key, agent_type, status, provider, model, owner_id, created_at) + VALUES (?,?,?,'predefined','active','test','test-model','owner',?), + (?,?,?,'predefined','active','test','test-model','owner',?)`, + safeRootAgentID, tenantID, safeRootKey, "2026-01-01T00:00:00Z", + recreatedOnlyAgentID, tenantID, recreatedOnlyKey, "2026-03-01T00:00:00Z", + ); err != nil { + t.Fatalf("insert temporal migration agents: %v", err) + } + + metadataTaskID := uuid.Must(uuid.NewV7()) + ambiguousTaskID := uuid.Must(uuid.NewV7()) + safeTaskID := uuid.Must(uuid.NewV7()) + recreatedTaskID := uuid.Must(uuid.NewV7()) + if _, err := db.Exec( + `INSERT INTO subagent_tasks + (id, tenant_id, parent_agent_key, subject, description, status, metadata, created_at) + VALUES (?,?,?,?,?,'queued',?,?), + (?,?,?,?,?,'queued','{}',?), + (?,?,?,?,?,'queued','{}',?), + (?,?,?,?,?,'queued','{}',?)`, + metadataTaskID, tenantID, rootAgentKey, "metadata", "metadata owner", + fmt.Sprintf(`{"root_agent_id":%q}`, oldRootAgentID.String()), "2026-04-01T00:00:00Z", + ambiguousTaskID, tenantID, rootAgentKey, "ambiguous", "ambiguous owner", "2026-04-01T00:00:00Z", + safeTaskID, tenantID, safeRootKey, "safe", "safe temporal owner", "2026-02-01T00:00:00Z", + recreatedTaskID, tenantID, recreatedOnlyKey, "recreated", "older than current owner", "2026-02-01T00:00:00Z", + ); err != nil { + t.Fatalf("insert legacy tasks: %v", err) + } + + if _, err := db.Exec(` + DROP TRIGGER trg_subagent_tasks_root_tenant_insert; + DROP TRIGGER trg_subagent_tasks_root_tenant_update; + DROP INDEX idx_subagent_tasks_root_archive; + DROP INDEX idx_subagent_tasks_root_session; + DROP INDEX idx_subagent_tasks_root_status; + ALTER TABLE subagent_tasks DROP COLUMN root_agent_id; + UPDATE schema_version SET version = 58; + `); err != nil { + t.Fatalf("prepare v58 schema: %v", err) + } + if err := EnsureSchema(db); err != nil { + t.Fatalf("migrate v58 to v59: %v", err) + } + + var metadataOwner sql.NullString + if err := db.QueryRow( + `SELECT root_agent_id FROM subagent_tasks WHERE id = ?`, metadataTaskID, + ).Scan(&metadataOwner); err != nil { + t.Fatalf("read metadata-owned task: %v", err) + } + if !metadataOwner.Valid || metadataOwner.String != oldRootAgentID.String() { + t.Fatalf("metadata owner = %q, want %s", metadataOwner.String, oldRootAgentID) + } + var ambiguousOwner sql.NullString + if err := db.QueryRow( + `SELECT root_agent_id FROM subagent_tasks WHERE id = ?`, ambiguousTaskID, + ).Scan(&ambiguousOwner); err != nil { + t.Fatalf("read ambiguous task: %v", err) + } + if ambiguousOwner.Valid { + t.Fatalf("ambiguous key owner = %q, want NULL", ambiguousOwner.String) + } + var safeOwner sql.NullString + if err := db.QueryRow( + `SELECT root_agent_id FROM subagent_tasks WHERE id = ?`, safeTaskID, + ).Scan(&safeOwner); err != nil { + t.Fatalf("read safely backfilled task: %v", err) + } + if !safeOwner.Valid || safeOwner.String != safeRootAgentID.String() { + t.Fatalf("safe owner = %q, want %s", safeOwner.String, safeRootAgentID) + } + var recreatedOwner sql.NullString + if err := db.QueryRow( + `SELECT root_agent_id FROM subagent_tasks WHERE id = ?`, recreatedTaskID, + ).Scan(&recreatedOwner); err != nil { + t.Fatalf("read recreated-owner task: %v", err) + } + if recreatedOwner.Valid { + t.Fatalf("new same-key agent inherited older task: owner=%q", recreatedOwner.String) + } + var version int + if err := db.QueryRow(`SELECT version FROM schema_version`).Scan(&version); err != nil { + t.Fatalf("read migrated schema version: %v", err) + } + if version != SchemaVersion { + t.Fatalf("schema version = %d, want %d", version, SchemaVersion) + } +} + +func TestSQLiteSubagentTaskStoreCompletedAtOnlyForTerminalStatus(t *testing.T) { + db := newHookTestDB(t) + tenantID, rootAgentID := seedHookTenantAgent(t, db) + ctx := sqliteTenantCtx(tenantID) + taskStore := NewSQLiteSubagentTaskStore(db) + + tests := []struct { + status string + terminal bool + }{ + {status: "new"}, + {status: "queued"}, + {status: "running"}, + {status: "waiting_child"}, + {status: "completed", terminal: true}, + {status: "failed", terminal: true}, + {status: "cancelled", terminal: true}, + } + + for _, tt := range tests { + t.Run(tt.status, func(t *testing.T) { + rootAgentKey := "root-" + tt.status + id := createSQLiteSubagentTask(t, taskStore, ctx, rootAgentID, rootAgentKey, "session-"+tt.status, "queued") + if err := taskStore.UpdateStatus(ctx, rootAgentID, id, tt.status, nil, 0, 0, 0); err != nil { + t.Fatalf("UpdateStatus(%q): %v", tt.status, err) + } + got, err := taskStore.Get(ctx, rootAgentID, id) + if err != nil { + t.Fatalf("Get(%q): %v", tt.status, err) + } + if (got.CompletedAt != nil) != tt.terminal { + t.Fatalf("status %q completed_at = %v, terminal=%v", tt.status, got.CompletedAt, tt.terminal) + } + }) + } +} + +func TestSQLiteSubagentTaskStoreRecoverInterrupted(t *testing.T) { + db := newHookTestDB(t) + tenantA, rootAID := seedHookTenantAgent(t, db) + tenantB, rootBID := seedHookTenantAgent(t, db) + ctxA := sqliteTenantCtx(tenantA) + ctxB := sqliteTenantCtx(tenantB) + taskStore := NewSQLiteSubagentTaskStore(db) + + queuedID := createSQLiteSubagentTask(t, taskStore, ctxA, rootAID, "root-a", "queued", "queued") + if _, err := db.Exec( + `UPDATE subagent_tasks SET completed_at = ? WHERE id = ?`, + time.Now().UTC().Add(-time.Hour).Format(time.RFC3339Nano), + queuedID, + ); err != nil { + t.Fatalf("seed malformed queued completed_at: %v", err) + } + runningID := createSQLiteSubagentTask(t, taskStore, ctxA, rootAID, "root-a", "running", "running") + waitingID := createSQLiteSubagentTask( + t, taskStore, ctxB, rootBID, "root-b", "waiting", "waiting_child", + ) + if err := taskStore.UpdateMetadata(ctxB, rootBID, waitingID, map[string]any{ + "completion_kind": "delegate", + "completion_media": []map[string]any{{ + "path": ".delegations/completed-before-crash/report.pdf", + "mime_type": "application/pdf", + }}, + }); err != nil { + t.Fatalf("record published artifact metadata: %v", err) + } + completedID := createSQLiteSubagentTask(t, taskStore, ctxB, rootBID, "root-b", "completed", "queued") + completedResult := "already completed" + if err := taskStore.UpdateStatus( + ctxB, rootBID, completedID, "completed", &completedResult, 1, 2, 3, + ); err != nil { + t.Fatalf("complete terminal task: %v", err) + } + + if _, err := taskStore.RecoverInterrupted(ctxA); err == nil { + t.Fatal("RecoverInterrupted accepted tenant-scoped context") + } + recoveryCtx := store.WithTenantID(context.Background(), store.MasterTenantID) + recovered, err := taskStore.RecoverInterrupted(recoveryCtx) + if err != nil { + t.Fatalf("RecoverInterrupted: %v", err) + } + if recovered != 3 { + t.Fatalf("RecoverInterrupted recovered %d tasks, want 3", recovered) + } + + for _, item := range []struct { + ctx context.Context + root uuid.UUID + id uuid.UUID + }{ + {ctx: ctxA, root: rootAID, id: queuedID}, + {ctx: ctxA, root: rootAID, id: runningID}, + {ctx: ctxB, root: rootBID, id: waitingID}, + } { + got, getErr := taskStore.Get(item.ctx, item.root, item.id) + if getErr != nil { + t.Fatalf("Get recovered task %s: %v", item.id, getErr) + } + if got.Status != "failed" || got.CompletedAt == nil || got.Result == nil || + !strings.Contains(*got.Result, "gateway stopped") { + t.Fatalf("recovered task %s = %#v", item.id, got) + } + if item.id == waitingID { + if got.Metadata["completion_kind"] != "delegate" || + got.Metadata["completion_media"] == nil { + t.Fatalf("published artifact metadata was lost: %#v", got.Metadata) + } + } + } + + completed, err := taskStore.Get(ctxB, rootBID, completedID) + if err != nil { + t.Fatalf("Get completed task: %v", err) + } + if completed.Status != "completed" || completed.Result == nil || + *completed.Result != completedResult { + t.Fatalf("completed task changed during recovery: %#v", completed) + } + + recovered, err = taskStore.RecoverInterrupted(recoveryCtx) + if err != nil { + t.Fatalf("RecoverInterrupted second pass: %v", err) + } + if recovered != 0 { + t.Fatalf("RecoverInterrupted second pass recovered %d tasks, want 0", recovered) + } +} + +func TestSQLiteSubagentTaskStoreArchiveIsScopedAndBounded(t *testing.T) { + db := newHookTestDB(t) + tenantA, rootAID := seedHookTenantAgent(t, db) + tenantB, tenantBRootID := seedHookTenantAgent(t, db) + ctxA := sqliteTenantCtx(tenantA) + ctxB := sqliteTenantCtx(tenantB) + taskStore := NewSQLiteSubagentTaskStore(db) + + const ( + rootA = "root-a" + rootB = "root-b" + ) + rootBID := uuid.Must(uuid.NewV7()) + if _, err := db.Exec( + `INSERT INTO agents (id, tenant_id, agent_key, agent_type, status, provider, model, owner_id) + VALUES (?,?,?,'predefined','active','test','test-model','owner')`, + rootBID, tenantA, rootB, + ); err != nil { + t.Fatalf("seed second root agent: %v", err) + } + var rootATasks []uuid.UUID + for i := 0; i < 3; i++ { + rootATasks = append(rootATasks, createSQLiteSubagentTask( + t, taskStore, ctxA, rootAID, rootA, fmt.Sprintf("session-a-%d", i), "queued", + )) + } + rootBTask := createSQLiteSubagentTask(t, taskStore, ctxA, rootBID, rootB, "session-b", "queued") + tenantBTask := createSQLiteSubagentTask(t, taskStore, ctxB, tenantBRootID, rootA, "session-other-tenant", "queued") + queuedTask := createSQLiteSubagentTask(t, taskStore, ctxA, rootAID, rootA, "session-queued", "queued") + + for _, item := range []struct { + ctx context.Context + root uuid.UUID + id uuid.UUID + }{ + {ctx: ctxA, root: rootAID, id: rootATasks[0]}, + {ctx: ctxA, root: rootAID, id: rootATasks[1]}, + {ctx: ctxA, root: rootAID, id: rootATasks[2]}, + {ctx: ctxA, root: rootBID, id: rootBTask}, + {ctx: ctxB, root: tenantBRootID, id: tenantBTask}, + } { + if err := taskStore.UpdateStatus(item.ctx, item.root, item.id, "completed", nil, 0, 0, 0); err != nil { + t.Fatalf("UpdateStatus(%s): %v", item.id, err) + } + } + + oldTime := time.Now().UTC().Add(-2 * time.Hour).Format(time.RFC3339Nano) + for _, id := range append(append([]uuid.UUID{}, rootATasks...), rootBTask, tenantBTask) { + if _, err := db.Exec(`UPDATE subagent_tasks SET completed_at = ? WHERE id = ?`, oldTime, id); err != nil { + t.Fatalf("backdate terminal task %s: %v", id, err) + } + } + // A non-terminal row must remain unarchived even if malformed legacy data + // happens to carry a completed_at value. + if _, err := db.Exec(`UPDATE subagent_tasks SET completed_at = ? WHERE id = ?`, oldTime, queuedTask); err != nil { + t.Fatalf("backdate queued task: %v", err) + } + + archived, err := taskStore.Archive(ctxA, rootAID, time.Hour, 2) + if err != nil { + t.Fatalf("Archive first batch: %v", err) + } + if archived != 2 { + t.Fatalf("Archive first batch affected %d rows, want 2", archived) + } + + assertSQLiteArchivedCount(t, db, tenantA, rootAID, 2) + assertSQLiteArchivedCount(t, db, tenantA, rootBID, 0) + assertSQLiteArchivedCount(t, db, tenantB, tenantBRootID, 0) + + archived, err = taskStore.Archive(ctxA, rootAID, time.Hour, 2) + if err != nil { + t.Fatalf("Archive second batch: %v", err) + } + if archived != 1 { + t.Fatalf("Archive second batch affected %d rows, want 1", archived) + } + assertSQLiteArchivedCount(t, db, tenantA, rootAID, 3) + + var queuedArchived sql.NullString + if err := db.QueryRow(`SELECT archived_at FROM subagent_tasks WHERE id = ?`, queuedTask).Scan(&queuedArchived); err != nil { + t.Fatalf("read queued archived_at: %v", err) + } + if queuedArchived.Valid { + t.Fatalf("queued task archived_at = %q, want NULL", queuedArchived.String) + } +} + +func assertSQLiteArchivedCount( + t *testing.T, db *sql.DB, tenantID, rootAgentID uuid.UUID, want int, +) { + t.Helper() + var got int + if err := db.QueryRow( + `SELECT COUNT(*) FROM subagent_tasks + WHERE tenant_id = ? AND root_agent_id = ? AND archived_at IS NOT NULL`, + tenantID, rootAgentID, + ).Scan(&got); err != nil { + t.Fatalf("count archived tasks for %s/%s: %v", tenantID, rootAgentID, err) + } + if got != want { + t.Fatalf("archived tasks for %s/%s = %d, want %d", tenantID, rootAgentID, got, want) + } +} diff --git a/internal/store/stores.go b/internal/store/stores.go index f2d59975..41ce5fda 100644 --- a/internal/store/stores.go +++ b/internal/store/stores.go @@ -40,6 +40,7 @@ type Stores struct { SkillEvolution SkillEvolutionStore SystemConfigs SystemConfigStore SubagentTasks SubagentTaskStore + SubagentTaskRecovery SubagentTaskRecoveryStore Vault VaultStore Episodic EpisodicStore EvolutionMetrics EvolutionMetricsStore diff --git a/internal/store/subagent_store.go b/internal/store/subagent_store.go index 6119dca7..09803f69 100644 --- a/internal/store/subagent_store.go +++ b/internal/store/subagent_store.go @@ -2,15 +2,39 @@ package store import ( "context" + "errors" "time" "github.com/google/uuid" ) +var ( + ErrSubagentRootAgentIDRequired = errors.New("subagent root agent ID required") + ErrSubagentTaskNotFound = errors.New("subagent task not found in owner scope") +) + +const ( + InterruptedSubagentTaskResult = "interrupted: gateway stopped before terminal completion was durably recorded" + // SubagentTaskStatusMaxLength matches the durable PG and SQLite schema contract. + SubagentTaskStatusMaxLength = 20 +) + +// IsTerminalSubagentTaskStatus reports whether a status ends the task +// lifecycle and therefore receives a completed_at timestamp. +func IsTerminalSubagentTaskStatus(status string) bool { + switch status { + case "completed", "failed", "cancelled": + return true + default: + return false + } +} + // SubagentTaskData represents a persisted subagent task for audit trail and cost attribution. type SubagentTaskData struct { BaseModel TenantID uuid.UUID `json:"tenant_id" db:"tenant_id"` + RootAgentID uuid.UUID `json:"root_agent_id" db:"root_agent_id"` ParentAgentKey string `json:"parent_agent_key" db:"parent_agent_key"` SessionKey *string `json:"session_key,omitempty" db:"session_key"` Subject string `json:"subject" db:"subject"` @@ -34,29 +58,39 @@ type SubagentTaskData struct { } // SubagentTaskStore persists subagent task lifecycle for audit trail and cost attribution. -// In-memory SubagentManager remains the source of truth for active operations; -// DB writes are fire-and-forget (non-blocking). +// In-memory SubagentManager remains the source of truth for active operations. type SubagentTaskStore interface { // Create persists a new subagent task at spawn time. Create(ctx context.Context, task *SubagentTaskData) error - // Get retrieves a single task by ID (tenant-scoped). - Get(ctx context.Context, id uuid.UUID) (*SubagentTaskData, error) + // Get retrieves a task owned by the tenant and immutable root-agent UUID. + Get(ctx context.Context, rootAgentID, id uuid.UUID) (*SubagentTaskData, error) // UpdateStatus updates status, result, iterations, and token counts on completion/failure. - UpdateStatus(ctx context.Context, id uuid.UUID, status string, result *string, iterations int, inputTokens, outputTokens int64) error + UpdateStatus(ctx context.Context, rootAgentID, id uuid.UUID, status string, result *string, iterations int, inputTokens, outputTokens int64) error - // ListByParent returns tasks for a parent agent key, optionally filtered by status. + // ListByParent returns tasks owned by a root-agent UUID, optionally filtered by status. // Empty statusFilter returns all statuses. Ordered by created_at DESC. - ListByParent(ctx context.Context, parentAgentKey string, statusFilter string) ([]SubagentTaskData, error) + ListByParent(ctx context.Context, rootAgentID uuid.UUID, statusFilter string) ([]SubagentTaskData, error) - // ListBySession returns tasks for a specific session key (tenant-scoped). - ListBySession(ctx context.Context, sessionKey string) ([]SubagentTaskData, error) + // ListBySession returns tasks for a session owned by the tenant and immutable root-agent UUID. + ListBySession(ctx context.Context, rootAgentID uuid.UUID, sessionKey string) ([]SubagentTaskData, error) - // Archive marks old completed/failed/cancelled tasks as archived. - // Returns the number of rows affected. - Archive(ctx context.Context, olderThan time.Duration) (int64, error) + // Archive marks at most limit old terminal tasks owned by the tenant and + // immutable root-agent UUID as archived. Returns the number of rows affected. + Archive(ctx context.Context, rootAgentID uuid.UUID, olderThan time.Duration, limit int) (int64, error) // UpdateMetadata merges metadata on an existing task. - UpdateMetadata(ctx context.Context, id uuid.UUID, metadata map[string]any) error + UpdateMetadata(ctx context.Context, rootAgentID, id uuid.UUID, metadata map[string]any) error +} + +// SubagentTaskRecoveryStore exposes the startup-only, cross-tenant maintenance +// operation separately from tenant/root-scoped task CRUD. +type SubagentTaskRecoveryStore interface { + // RecoverInterrupted marks every non-terminal task from the previous + // process as failed. It is a cross-tenant startup operation and must run + // before the gateway accepts new child runs. The current gateway + // architecture is single-process; multi-replica deployments would need + // coordinated ownership before invoking this recovery. + RecoverInterrupted(ctx context.Context) (int64, error) } diff --git a/internal/tools/announce_queue.go b/internal/tools/announce_queue.go index 521b63c4..98d9718c 100644 --- a/internal/tools/announce_queue.go +++ b/internal/tools/announce_queue.go @@ -1,6 +1,8 @@ package tools import ( + "context" + "errors" "fmt" "strings" "sync" @@ -11,17 +13,23 @@ import ( "github.com/nextlevelbuilder/goclaw/internal/bus" ) +var ErrAnnounceQueueDrainTimeout = errors.New("announce_queue_drain_timeout") + // AnnounceQueueItem represents a single subagent result waiting to be announced. type AnnounceQueueItem struct { - SubagentID string - Label string - Status string // "completed", "failed", "cancelled" - Result string - Media []bus.MediaFile // media files from tool results - Runtime time.Duration - Iterations int - InputTokens int64 - OutputTokens int64 + SubagentID string + CompletionID uuid.UUID + DurablyPersisted bool + ParentTaskID string + Depth int + Label string + Status string // "completed", "failed", "cancelled" + Result string + Media []bus.MediaFile // media files from tool results + Runtime time.Duration + Iterations int + InputTokens int64 + OutputTokens int64 } // AnnounceMetadata carries origin info for routing the batched announce. @@ -31,10 +39,11 @@ type AnnounceMetadata struct { OriginPeerKind string OriginLocalKey string // composite key with topic/thread suffix for routing OriginUserID string - OriginSenderID string // real acting sender; preserves permission attribution through re-ingress (#915) - OriginRole string // caller's RBAC role; bypasses per-user grants for admin/operator/owner (#915) - OriginSessionKey string // exact parent session key (WS uses non-standard format) + OriginSenderID string // real acting sender; preserves permission attribution through re-ingress (#915) + OriginRole string // caller's RBAC role; bypasses per-user grants for admin/operator/owner (#915) + OriginSessionKey string // exact parent session key (WS uses non-standard format) OriginTenantID uuid.UUID // parent tenant for announce routing + RootAgentID uuid.UUID ParentAgent string OriginTraceID string // parent trace UUID for announce linking OriginRootSpanID string // parent agent's root span UUID @@ -47,7 +56,9 @@ type AnnounceQueue struct { queues map[string]*sessionQueue // session key → queue debounce time.Duration // default 1000ms cap int // max items per session before immediate drain (default 20) - onDrain func(sessionKey string, items []AnnounceQueueItem, meta AnnounceMetadata) + onDrain func(sessionKey string, items []AnnounceQueueItem, meta AnnounceMetadata) + closed bool + drainWG sync.WaitGroup } type sessionQueue struct { @@ -81,6 +92,9 @@ func NewAnnounceQueue( func (aq *AnnounceQueue) Enqueue(sessionKey string, item AnnounceQueueItem, meta AnnounceMetadata) { aq.mu.Lock() defer aq.mu.Unlock() + if aq.closed { + return + } sq, ok := aq.queues[sessionKey] if !ok { @@ -98,7 +112,7 @@ func (aq *AnnounceQueue) Enqueue(sessionKey string, item AnnounceQueueItem, meta items := sq.items sqMeta := sq.meta delete(aq.queues, sessionKey) - go aq.drain(sessionKey, items, sqMeta) + aq.startDrainLocked(sessionKey, items, sqMeta) return } @@ -116,12 +130,62 @@ func (aq *AnnounceQueue) Enqueue(sessionKey string, item AnnounceQueueItem, meta items := sq.items sqMeta := sq.meta delete(aq.queues, sessionKey) + aq.startDrainLocked(sessionKey, items, sqMeta) aq.mu.Unlock() - - aq.drain(sessionKey, items, sqMeta) }) } +func (aq *AnnounceQueue) startDrainLocked( + sessionKey string, + items []AnnounceQueueItem, + meta AnnounceMetadata, +) { + if aq.closed { + return + } + aq.drainWG.Add(1) + go func() { + defer aq.drainWG.Done() + aq.drain(sessionKey, items, meta) + }() +} + +// CloseContext closes intake, drops pending debounce batches, and waits for any +// drain callback that already started. Dropping pending parent-resume messages +// during process shutdown avoids starting new agent runs against dependencies +// that are being torn down; task terminal state remains durable. +func (aq *AnnounceQueue) CloseContext(ctx context.Context) error { + if aq == nil { + return nil + } + if ctx == nil { + ctx = context.Background() + } + aq.mu.Lock() + if !aq.closed { + aq.closed = true + for _, queue := range aq.queues { + if queue.timer != nil { + queue.timer.Stop() + } + } + clear(aq.queues) + } + aq.mu.Unlock() + + done := make(chan struct{}) + go func() { + aq.drainWG.Wait() + close(done) + }() + select { + case <-done: + return nil + case <-ctx.Done(): + return fmt.Errorf("%w: %v", ErrAnnounceQueueDrainTimeout, ctx.Err()) + } +} + // drain merges items into a single announce message and calls onDrain. func (aq *AnnounceQueue) drain(sessionKey string, items []AnnounceQueueItem, meta AnnounceMetadata) { if len(items) == 0 || aq.onDrain == nil { @@ -147,11 +211,11 @@ func FormatBatchedAnnounce(items []AnnounceQueueItem, roster SubagentRoster) str replyInstruction := BuildReplyInstruction(roster) return fmt.Sprintf( - "[System Message] A subagent task %q just %s.\n\n"+ + "[System Message] A subagent task %q just %s (task=%s, parent=%s, depth=%d).\n\n"+ "Result:\n%s\n\n"+ "Stats: runtime %s, iterations %d, tokens %d in / %d out\n\n"+ "%s", - item.Label, statusLabel, item.Result, + item.Label, statusLabel, item.SubagentID, item.ParentTaskID, item.Depth, item.Result, item.Runtime.Round(time.Millisecond), item.Iterations, item.InputTokens, item.OutputTokens, replyInstruction, @@ -171,8 +235,8 @@ func FormatBatchedAnnounce(items []AnnounceQueueItem, roster SubagentRoster) str } sb.WriteString(fmt.Sprintf( - "\n---\nTask #%d: %q %s (runtime %s, iterations %d, tokens %d/%d)\nResult: %s\n", - i+1, item.Label, statusLabel, + "\n---\nTask #%d: %q %s (task=%s, parent=%s, depth=%d, runtime %s, iterations %d, tokens %d/%d)\nResult: %s\n", + i+1, item.Label, statusLabel, item.SubagentID, item.ParentTaskID, item.Depth, item.Runtime.Round(time.Millisecond), item.Iterations, item.InputTokens, item.OutputTokens, item.Result, @@ -188,11 +252,12 @@ func FormatBatchedAnnounce(items []AnnounceQueueItem, roster SubagentRoster) str // buildReplyInstruction generates the instruction block for the parent LLM, // including a deterministic roster of all subagent tasks with their statuses. func BuildReplyInstruction(roster SubagentRoster) string { - // Count running tasks from roster - running := 0 - for _, e := range roster.Entries { - if e.Status == TaskStatusRunning { - running++ + running := roster.Active + if running == 0 { + for _, entry := range roster.Entries { + if entry.Status == TaskStatusRunning { + running++ + } } } @@ -200,8 +265,10 @@ func BuildReplyInstruction(roster SubagentRoster) string { var rosterBlock string if len(roster.Entries) > 0 { var sb strings.Builder - sb.WriteString(fmt.Sprintf("Subagent roster (%d spawned, max %d per agent):\n", - roster.Total, roster.MaxPerAgent)) + sb.WriteString(fmt.Sprintf( + "Subagent roster (%d active / %d max; %d retained total):\n", + running, roster.MaxPerAgent, roster.Total, + )) for _, e := range roster.Entries { sb.WriteString(fmt.Sprintf(" [%-9s] %s\n", e.Status, e.Label)) } diff --git a/internal/tools/announce_queue_lifecycle_test.go b/internal/tools/announce_queue_lifecycle_test.go new file mode 100644 index 00000000..fb7b732a --- /dev/null +++ b/internal/tools/announce_queue_lifecycle_test.go @@ -0,0 +1,50 @@ +package tools + +import ( + "context" + "errors" + "testing" + "time" +) + +func TestAnnounceQueueCloseDropsPendingDebounce(t *testing.T) { + drained := make(chan struct{}, 1) + queue := NewAnnounceQueue(20, 20, func(string, []AnnounceQueueItem, AnnounceMetadata) { + drained <- struct{}{} + }) + queue.Enqueue("session", AnnounceQueueItem{SubagentID: "task"}, AnnounceMetadata{}) + if err := queue.CloseContext(context.Background()); err != nil { + t.Fatal(err) + } + select { + case <-drained: + t.Fatal("pending announce drained after queue close") + case <-time.After(50 * time.Millisecond): + } +} + +func TestAnnounceQueueCloseContextBoundsActiveDrain(t *testing.T) { + started := make(chan struct{}) + release := make(chan struct{}) + queue := NewAnnounceQueue(1, 1, func(string, []AnnounceQueueItem, AnnounceMetadata) { + close(started) + <-release + }) + queue.Enqueue("session", AnnounceQueueItem{SubagentID: "task"}, AnnounceMetadata{}) + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("announce drain did not start") + } + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + err := queue.CloseContext(ctx) + if !errors.Is(err, ErrAnnounceQueueDrainTimeout) { + t.Fatalf("CloseContext error = %v, want typed drain timeout", err) + } + close(release) + if err := queue.CloseContext(context.Background()); err != nil { + t.Fatalf("CloseContext retry: %v", err) + } +} diff --git a/internal/tools/async_completion_delivery.go b/internal/tools/async_completion_delivery.go new file mode 100644 index 00000000..1e338ad2 --- /dev/null +++ b/internal/tools/async_completion_delivery.go @@ -0,0 +1,93 @@ +package tools + +import ( + "context" + "time" + + "github.com/nextlevelbuilder/goclaw/internal/bus" +) + +const asyncCompletionAttempts = 3 + +const ( + asyncTerminalPersistenceAttempts = 12 + asyncPersistenceAttemptTimeout = 2 * time.Second + asyncPersistenceMaxBackoff = time.Second +) + +// PublishAsyncCompletion retries only after child execution has released its +// admission lease. The durable task row remains the fallback if every attempt +// finds the inbound queue saturated. +func PublishAsyncCompletion( + ctx context.Context, + msgBus *bus.MessageBus, + message bus.InboundMessage, +) bool { + if msgBus == nil { + return true + } + if ctx == nil { + ctx = context.Background() + } + for attempt := range asyncCompletionAttempts { + if msgBus.TryPublishInbound(message) { + return true + } + if attempt == asyncCompletionAttempts-1 { + break + } + timer := time.NewTimer(time.Duration(attempt+1) * 10 * time.Millisecond) + select { + case <-ctx.Done(): + timer.Stop() + return false + case <-timer.C: + } + } + return false +} + +func retryAsyncPersistence( + ctx context.Context, + operation func(context.Context) error, +) error { + return retryPersistence(ctx, asyncCompletionAttempts, operation) +} + +func retryTerminalPersistence( + ctx context.Context, + operation func(context.Context) error, +) error { + return retryPersistence(ctx, asyncTerminalPersistenceAttempts, operation) +} + +func retryPersistence( + ctx context.Context, + attempts int, + operation func(context.Context) error, +) error { + if ctx == nil { + ctx = context.Background() + } + var err error + for attempt := range attempts { + attemptCtx, cancel := context.WithTimeout(ctx, asyncPersistenceAttemptTimeout) + err = operation(attemptCtx) + cancel() + if err == nil { + return nil + } + if attempt == attempts-1 { + break + } + backoff := min(time.Duration(1< 500*time.Millisecond { + t.Fatalf("running persistence deadline = %s, want at most 500ms", remaining) + } + default: + t.Fatal("running persistence was not attempted") + } + if attempts := runningAttempts.Load(); attempts != 2 { + t.Fatalf("running persistence attempts = %d, want one per accepted delegation", attempts) + } +} + +func TestCompletionMediaDescriptorsNeverPersistHostPaths(t *testing.T) { + workspace := t.TempDir() + inside := filepath.Join(workspace, ".uploads", "report.pdf") + outside := filepath.Join(t.TempDir(), "secret.txt") + descriptors := completionMediaDescriptors([]bus.MediaFile{ + {Path: inside, MimeType: "application/pdf", Filename: "report.pdf"}, + {Path: outside, MimeType: "text/plain", Filename: "secret.txt"}, + }, workspace, "") + + if len(descriptors) != 1 || descriptors[0].Path != ".uploads/report.pdf" { + t.Fatalf("completion media = %#v, want one logical workspace path", descriptors) + } + encoded, err := json.Marshal(descriptors) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(encoded), workspace) || strings.Contains(string(encoded), outside) { + t.Fatalf("completion media leaked host path: %s", encoded) + } + if payload := persistedCompletionMediaPayload([]map[string]any{{ + "path": outside, + }}); len(payload) != 0 { + t.Fatalf("absolute persisted media was returned: %#v", payload) + } +} diff --git a/internal/tools/async_completion_media.go b/internal/tools/async_completion_media.go new file mode 100644 index 00000000..d4c24816 --- /dev/null +++ b/internal/tools/async_completion_media.go @@ -0,0 +1,80 @@ +package tools + +import ( + "encoding/json" + "path/filepath" + "strings" + + "github.com/nextlevelbuilder/goclaw/internal/bus" +) + +const asyncCompletionMediaKey = "completion_media" + +type persistedCompletionMedia struct { + Path string `json:"path"` + MimeType string `json:"mime_type,omitempty"` + Filename string `json:"filename,omitempty"` + Caption string `json:"caption,omitempty"` +} + +// completionMediaDescriptors converts runtime host paths into workspace-safe +// logical paths. Outside-workspace media is intentionally omitted. +func completionMediaDescriptors( + media []bus.MediaFile, + workspace, logicalPrefix string, +) []persistedCompletionMedia { + if len(media) == 0 || workspace == "" { + return nil + } + descriptors := make([]persistedCompletionMedia, 0, len(media)) + for _, item := range media { + rawPath := item.Path + if !filepath.IsAbs(rawPath) { + rawPath = filepath.Join(workspace, rawPath) + } + absolutePath, err := filepath.Abs(rawPath) + if err != nil { + continue + } + relativePath, err := filepath.Rel(workspace, absolutePath) + if err != nil { + continue + } + logicalPath := filepath.ToSlash(relativePath) + if logicalPrefix != "" { + logicalPath = strings.TrimSuffix(logicalPrefix, "/") + "/" + logicalPath + } + normalized, err := validateArtifactRelativePath(logicalPath) + if err != nil { + continue + } + descriptors = append(descriptors, persistedCompletionMedia{ + Path: normalized, + MimeType: item.MimeType, + Filename: item.Filename, + Caption: item.Caption, + }) + } + return descriptors +} + +func persistedCompletionMediaPayload(raw any) []persistedCompletionMedia { + encoded, err := json.Marshal(raw) + if err != nil { + return nil + } + var decoded []persistedCompletionMedia + if err := json.Unmarshal(encoded, &decoded); err != nil { + return nil + } + safe := decoded[:0] + for _, item := range decoded { + normalized, err := validateArtifactRelativePath(item.Path) + if err != nil { + continue + } + item.Path = normalized + safe = append(safe, item) + } + return safe +} diff --git a/internal/tools/context_keys.go b/internal/tools/context_keys.go index 2664b871..65f21e33 100644 --- a/internal/tools/context_keys.go +++ b/internal/tools/context_keys.go @@ -2,12 +2,14 @@ package tools import ( "context" + "fmt" "log/slog" "maps" "sync" "github.com/google/uuid" + orchestration "github.com/nextlevelbuilder/goclaw/internal/childrun" "github.com/nextlevelbuilder/goclaw/internal/config" "github.com/nextlevelbuilder/goclaw/internal/sandbox" "github.com/nextlevelbuilder/goclaw/internal/store" @@ -33,9 +35,76 @@ const ( ctxAgentPolicy toolContextKey = "tool_agent_policy" // per-agent tool policy for MCP bridge enforcement ctxSessionKey toolContextKey = "tool_session_key" // origin session key for announce routing ctxRunKind toolContextKey = "tool_run_kind" // "notification", "announce", "delegation" + ctxSubagentScope toolContextKey = "subagent_task_scope" + ctxSubagentTaskID toolContextKey = "subagent_task_id" + ctxSubagentDepth toolContextKey = "subagent_depth" + ctxChildRunLease toolContextKey = "child_run_lease" ctxTelegramManagerPermissions toolContextKey = "telegram_manager_permissions" ) +func withSubagentExecution(ctx context.Context, scope TaskScope, taskID string, depth int, lease *orchestration.ChildRunLease) context.Context { + ctx = context.WithValue(ctx, ctxSubagentScope, scope) + ctx = context.WithValue(ctx, ctxSubagentTaskID, taskID) + ctx = context.WithValue(ctx, ctxSubagentDepth, depth) + return context.WithValue(ctx, ctxChildRunLease, lease) +} + +func childRunLeaseFromContext(ctx context.Context) *orchestration.ChildRunLease { + lease, _ := ctx.Value(ctxChildRunLease).(*orchestration.ChildRunLease) + return lease +} + +// withDelegatedAgentExecution starts a fresh semantic spawn tree for the target +// agent while retaining the admission lease used by synchronous continuations. +func withDelegatedAgentExecution(ctx context.Context, lease *orchestration.ChildRunLease) context.Context { + ctx = context.WithValue(ctx, ctxSubagentScope, TaskScope{}) + ctx = context.WithValue(ctx, ctxSubagentTaskID, "") + ctx = context.WithValue(ctx, ctxSubagentDepth, 0) + ctx = WithSubagentConfig(ctx, nil) + return context.WithValue(ctx, ctxChildRunLease, lease) +} + +func subagentScopeFromContext(ctx context.Context) TaskScope { + if scope, ok := ctx.Value(ctxSubagentScope).(TaskScope); ok { + if scope.TenantID != uuid.Nil || scope.RootAgentID != uuid.Nil || scope.RootAgentKey != "" { + return scope + } + } + return TaskScope{ + TenantID: store.TenantIDFromContext(ctx), + RootAgentID: store.AgentIDFromContext(ctx), + RootAgentKey: ToolAgentKeyFromCtx(ctx), + } +} + +func subagentTaskIDFromContext(ctx context.Context) string { + id, _ := ctx.Value(ctxSubagentTaskID).(string) + return id +} + +func subagentDepthFromContext(ctx context.Context, fallback int) int { + if depth, ok := ctx.Value(ctxSubagentDepth).(int); ok { + return depth + } + return fallback +} + +// childRunContinuationLineage returns structural admission lineage. It is +// intentionally separate from semantic sub-agent depth, which resets at every +// Agent Link delegation boundary. +func childRunContinuationLineage( + ctx context.Context, + fallbackParentTaskID string, + fallbackDepth int, +) (string, int) { + if lease := childRunLeaseFromContext(ctx); lease != nil { + if parentTaskID, parentDepth, ok := lease.ContinuationParent(); ok { + return parentTaskID, parentDepth + 1 + } + } + return fallbackParentTaskID, fallbackDepth +} + // ctxRateLimitOverride carries a per-agent tool rate limit (calls/hour) that // overrides the global tools.rate_limit_per_hour. 0 means "use the global". const ctxRateLimitOverride toolContextKey = "tool_rate_limit_override" @@ -53,6 +122,13 @@ type MediaPathLoader interface { LoadPath(id string) (string, error) } +// MediaPathRootProvider optionally exposes the managed root that authorizes +// legacy media returned by MediaPathLoader. Implementations must return a +// stable root; callers still validate containment and regular-file semantics. +type MediaPathRootProvider interface { + MediaRootPath() string +} + func WithToolChannel(ctx context.Context, channel string) context.Context { return context.WithValue(ctx, ctxChannel, channel) } @@ -137,7 +213,7 @@ func WithToolWorkspace(ctx context.Context, ws string) context.Context { } func ToolWorkspaceFromCtx(ctx context.Context) string { - if v, _ := ctx.Value(ctxWorkspace).(string); v != "" { + if v, ok := ctx.Value(ctxWorkspace).(string); ok { return v } if rc := store.RunContextFromCtx(ctx); rc != nil { @@ -374,7 +450,8 @@ func WithSubagentConfig(ctx context.Context, cfg *config.SubagentsConfig) contex } func SubagentConfigFromCtx(ctx context.Context) *config.SubagentsConfig { - if v, _ := ctx.Value(ctxSubagentCfg).(*config.SubagentsConfig); v != nil { + if raw := ctx.Value(ctxSubagentCfg); raw != nil { + v, _ := raw.(*config.SubagentsConfig) return v } if rc := store.RunContextFromCtx(ctx); rc != nil { @@ -432,7 +509,7 @@ func WithToolTeamID(ctx context.Context, teamID string) context.Context { // ToolTeamIDFromCtx returns the dispatching team's ID from context. func ToolTeamIDFromCtx(ctx context.Context) string { - if v, _ := ctx.Value(ctxTeamID).(string); v != "" { + if v, ok := ctx.Value(ctxTeamID).(string); ok { return v } if rc := store.RunContextFromCtx(ctx); rc != nil { @@ -453,7 +530,7 @@ func WithToolTeamWorkspace(ctx context.Context, dir string) context.Context { // ToolTeamWorkspaceFromCtx returns the team shared workspace directory path. func ToolTeamWorkspaceFromCtx(ctx context.Context) string { - if v, _ := ctx.Value(ctxTeamWorkspace).(string); v != "" { + if v, ok := ctx.Value(ctxTeamWorkspace).(string); ok { return v } if rc := store.RunContextFromCtx(ctx); rc != nil { @@ -495,7 +572,7 @@ func WithTeamTaskID(ctx context.Context, taskID string) context.Context { // TeamTaskIDFromCtx returns the delegation's team task ID from context. func TeamTaskIDFromCtx(ctx context.Context) string { - if v, _ := ctx.Value(ctxTeamTaskID).(string); v != "" { + if v, ok := ctx.Value(ctxTeamTaskID).(string); ok { return v } if rc := store.RunContextFromCtx(ctx); rc != nil { @@ -518,7 +595,7 @@ func WithDelegationID(ctx context.Context, id string) context.Context { // DelegationIDFromCtx returns the active delegation ID. Falls back to // RunContext when no explicit context key is present. func DelegationIDFromCtx(ctx context.Context) string { - if v, _ := ctx.Value(ctxDelegationID).(string); v != "" { + if v, ok := ctx.Value(ctxDelegationID).(string); ok { return v } if rc := store.RunContextFromCtx(ctx); rc != nil { @@ -539,7 +616,7 @@ func WithLeaderAgentID(ctx context.Context, id string) context.Context { // LeaderAgentIDFromCtx returns the leader's agent UUID string from context. func LeaderAgentIDFromCtx(ctx context.Context) string { - if v, _ := ctx.Value(ctxLeaderAgentID).(string); v != "" { + if v, ok := ctx.Value(ctxLeaderAgentID).(string); ok { return v } if rc := store.RunContextFromCtx(ctx); rc != nil { @@ -820,7 +897,7 @@ func WithTenantAllowedPaths(ctx context.Context, paths []string) context.Context // TenantAllowedPathsFromCtx returns tenant-specific allowed paths from context. // Falls back to RunContext for subagent inheritance. func TenantAllowedPathsFromCtx(ctx context.Context) []string { - if v, _ := ctx.Value(ctxTenantAllowedPaths).([]string); len(v) > 0 { + if v, ok := ctx.Value(ctxTenantAllowedPaths).([]string); ok { return v } if rc := store.RunContextFromCtx(ctx); rc != nil { @@ -828,3 +905,31 @@ func TenantAllowedPathsFromCtx(ctx context.Context) []string { } return nil } + +const ctxDelegationArtifactInputs toolContextKey = "tool_delegation_artifact_inputs" + +// WithDelegationArtifactInputs installs the runtime-only host root backing the +// logical read-only inputs/ alias for an admitted Agent Link run. +func WithDelegationArtifactInputs(ctx context.Context, root string) context.Context { + return context.WithValue(ctx, ctxDelegationArtifactInputs, root) +} + +// DelegationArtifactInputsFromCtx returns the runtime-only staged-input root. +// It is deliberately not copied into RunContext, prompts, traces, or results. +func DelegationArtifactInputsFromCtx(ctx context.Context) string { + root, _ := ctx.Value(ctxDelegationArtifactInputs).(string) + return root +} + +// IsDelegationArtifactRun reports whether filesystem/media tools are executing +// inside an isolated Agent Link artifact exchange. +func IsDelegationArtifactRun(ctx context.Context) bool { + return DelegationIDFromCtx(ctx) != "" && DelegationArtifactInputsFromCtx(ctx) != "" +} + +func validateDelegationChildRunMode(ctx context.Context, operation, mode string) error { + if IsDelegationArtifactRun(ctx) && mode != "sync" { + return fmt.Errorf("%s mode %q is not allowed inside an Agent Link artifact run; use mode=\"sync\"", operation, mode) + } + return nil +} diff --git a/internal/tools/create_image.go b/internal/tools/create_image.go index 3e9cd5dd..db6ee3a8 100644 --- a/internal/tools/create_image.go +++ b/internal/tools/create_image.go @@ -12,6 +12,7 @@ import ( "net/http" "os" "path/filepath" + "strconv" "strings" "time" @@ -78,7 +79,11 @@ func (t *CreateImageTool) resolveReferenceImages(ctx context.Context, args map[s } seenPaths[path] = true - ext := strings.ToLower(filepath.Ext(path)) + resolved, err := resolveStructuredMediaPath(ctx, path, "reference image") + if err != nil { + return nil, err + } + ext := strings.ToLower(filepath.Ext(resolved)) mimeTypes := map[string]string{ ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".png": "image/png", ".gif": "image/gif", @@ -88,14 +93,6 @@ func (t *CreateImageTool) resolveReferenceImages(ctx context.Context, args map[s if !ok { mime = "image/png" } - workspace := ToolWorkspaceFromCtx(ctx) - resolved, err := resolvePathWithAllowed(path, workspace, effectiveRestrict(ctx, true), allowedWithTeamWorkspace(ctx, nil)) - if err != nil { - return nil, fmt.Errorf("invalid reference image path: %w", err) - } - if err := checkDeniedPath(resolved, workspace, nil); err != nil { - return nil, err - } data, err := os.ReadFile(resolved) if err != nil { return nil, fmt.Errorf("failed to read reference image file: %w", err) @@ -136,18 +133,25 @@ func (t *CreateImageTool) resolveReferenceImages(ctx context.Context, args map[s seenIDs[id] = true images := MediaImagesFromCtx(ctx) - if len(images) == 0 { - return nil, fmt.Errorf("no images available in conversation context") - } var img providers.ImageContent - if id == "latest" { + ref, refErr := resolveImageMediaRef(ctx, id) + if refErr == nil && ref.Path != "" { + fileImages, err := (&ReadImageTool{}).loadImageFromPath(ctx, ref.Path) + if err != nil { + return nil, fmt.Errorf("failed to resolve reference image media_id %q: %w", id, err) + } + img = fileImages[0] + } else if id == "latest" && len(images) > 0 { img = images[len(images)-1] } else { - var idx int - if _, err := fmt.Sscanf(id, "%d", &idx); err == nil && idx >= 0 && idx < len(images) { + idx, err := strconv.Atoi(id) + if err == nil && idx >= 0 && idx < len(images) { img = images[idx] } else { - img = images[len(images)-1] + if refErr != nil { + return nil, refErr + } + return nil, fmt.Errorf("image media_id %q has no accessible workspace path", id) } } dataBytes, _ := base64.StdEncoding.DecodeString(img.Data) @@ -226,9 +230,9 @@ func (t *CreateImageTool) Parameters() map[string]any { "items": map[string]any{ "type": "object", "properties": map[string]any{ - "path": map[string]any{"type": "string", "description": "Workspace file path to a reference image."}, + "path": map[string]any{"type": "string", "description": "Logical workspace path to a reference image, such as .uploads/photo.jpg or delegation input inputs/photo.jpg. Never guess an absolute path."}, "url": map[string]any{"type": "string", "description": "HTTP/HTTPS URL of a reference image."}, - "id": map[string]any{"type": "string", "description": "Media ID of a reference image from the chat."}, + "id": map[string]any{"type": "string", "description": "Exact image ID from a tag, or 'latest', for in-process tool execution. Claude CLI/MCP callers should use path."}, "strength": map[string]any{"type": "number", "description": "Reference strength (0.0 to 1.0) specific to this image."}, "description": map[string]any{"type": "string", "description": "Description of the role or content of this reference image (e.g. 'Lâm', 'Quân')."}, }, diff --git a/internal/tools/create_image_native_path_test.go b/internal/tools/create_image_native_path_test.go index 397a6196..a0958e40 100644 --- a/internal/tools/create_image_native_path_test.go +++ b/internal/tools/create_image_native_path_test.go @@ -396,6 +396,59 @@ func TestCreateImageTool_ResolveReferenceImage_ID(t *testing.T) { } } +func TestCreateImageTool_ResolveReferenceImage_ExactID(t *testing.T) { + workspace := t.TempDir() + firstPath := filepath.Join(workspace, "first.png") + secondPath := filepath.Join(workspace, "second.png") + firstData := []byte("first-image") + secondData := []byte("second-image") + if err := os.WriteFile(firstPath, firstData, 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(secondPath, secondData, 0644); err != nil { + t.Fatal(err) + } + + ctx := WithToolWorkspace(context.Background(), workspace) + ctx = WithMediaImageRefs(ctx, []providers.MediaRef{ + {ID: "image-first", Kind: "image", MimeType: "image/png", Path: firstPath}, + {ID: "image-second", Kind: "image", MimeType: "image/png", Path: secondPath}, + }) + + got, err := NewCreateImageTool(nil).resolveReferenceImages(ctx, map[string]any{ + "ref_images": []any{map[string]any{"id": "image-first"}}, + }) + if err != nil { + t.Fatalf("resolve exact media ID: %v", err) + } + if len(got) != 1 || string(got[0].Data) != string(firstData) { + t.Fatalf("resolved images = %#v, want exact first image", got) + } +} + +func TestCreateImageTool_ResolveReferenceImage_UnknownIDFails(t *testing.T) { + workspace := t.TempDir() + imagePath := filepath.Join(workspace, "known.png") + if err := os.WriteFile(imagePath, []byte("known-image"), 0644); err != nil { + t.Fatal(err) + } + ctx := WithToolWorkspace(context.Background(), workspace) + ctx = WithMediaImages(ctx, []providers.ImageContent{{ + MimeType: "image/png", + Data: base64.StdEncoding.EncodeToString([]byte("fallback-must-not-be-used")), + }}) + ctx = WithMediaImageRefs(ctx, []providers.MediaRef{{ + ID: "known-id", Kind: "image", MimeType: "image/png", Path: imagePath, + }}) + + _, err := NewCreateImageTool(nil).resolveReferenceImages(ctx, map[string]any{ + "ref_images": []any{map[string]any{"id": "missing-id"}}, + }) + if err == nil || !strings.Contains(err.Error(), `media_id "missing-id" not found`) { + t.Fatalf("unknown media ID error = %v", err) + } +} + func TestCreateImageTool_MultipleReferenceImages_MixedSources(t *testing.T) { tmpDir := t.TempDir() refFile := filepath.Join(tmpDir, "ref.png") diff --git a/internal/tools/credentialed_exec.go b/internal/tools/credentialed_exec.go index bcf7cd0e..852408fb 100644 --- a/internal/tools/credentialed_exec.go +++ b/internal/tools/credentialed_exec.go @@ -727,7 +727,7 @@ func (t *ExecTool) executeCredentialedSandbox(ctx context.Context, absPath strin return ErrorResult(fmt.Sprintf("credentialed sandbox path mapping: %v", cwdErr)) } - sb, err := t.sandboxMgr.Get(ctx, sandboxKey, mountWorkspace, SandboxConfigFromCtx(ctx)) + sb, err := acquireToolSandbox(ctx, t.sandboxMgr, sandboxKey, mountWorkspace) if err != nil { slog.Warn("security.credentialed_exec_sandbox_unavailable", "binary", absPath, "error", err) diff --git a/internal/tools/delegate_completion_ledger.go b/internal/tools/delegate_completion_ledger.go new file mode 100644 index 00000000..3f45b730 --- /dev/null +++ b/internal/tools/delegate_completion_ledger.go @@ -0,0 +1,238 @@ +package tools + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "time" + + "github.com/google/uuid" + + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +const delegateCompletionTargetKey = "target_agent_key" + +const delegateRunningPersistenceTimeout = 250 * time.Millisecond + +func (t *DelegateTool) createDelegateCompletion(ctx context.Context, req DelegateRequest) error { + if t.taskStore == nil { + return nil + } + delegationID, err := uuid.Parse(req.DelegationID) + if err != nil { + return fmt.Errorf("invalid delegation ID: %w", err) + } + tenantID := parseUUIDOrNil(req.TenantID) + if tenantID == uuid.Nil || req.FromAgentID == uuid.Nil { + return fmt.Errorf("delegation completion requires tenant and source agent") + } + var sessionKey, originChannel, originChatID, originPeerKind, originUserID *string + if req.SessionKey != "" { + sessionKey = &req.SessionKey + } + if req.Channel != "" { + originChannel = &req.Channel + } + if req.ChatID != "" { + originChatID = &req.ChatID + } + if req.PeerKind != "" { + originPeerKind = &req.PeerKind + } + if req.UserID != "" { + originUserID = &req.UserID + } + data := &store.SubagentTaskData{ + BaseModel: store.BaseModel{ID: delegationID}, + TenantID: tenantID, + RootAgentID: req.FromAgentID, + ParentAgentKey: req.FromAgentKey, + SessionKey: sessionKey, + Subject: "Delegate to " + req.ToAgentKey, + Description: req.Task, + Status: TaskStatusQueued, + Depth: subagentDepthFromContext(ctx, 0) + 1, + OriginChannel: originChannel, + OriginChatID: originChatID, + OriginPeerKind: originPeerKind, + OriginUserID: originUserID, + Metadata: map[string]any{ + asyncCompletionKindKey: asyncCompletionKindDelegate, + delegateCompletionTargetKey: req.ToAgentKey, + asyncCompletionDeliveryKey: asyncCompletionDeliveryPending, + }, + } + dbCtx := store.WithTenantID(context.Background(), tenantID) + if err := retryAsyncPersistence(dbCtx, func(attemptCtx context.Context) error { + return t.taskStore.Create(attemptCtx, data) + }); err != nil { + return fmt.Errorf("persist accepted delegation %s: %w", req.DelegationID, err) + } + return nil +} + +func (t *DelegateTool) updateDelegateCompletion( + req DelegateRequest, + status string, + result *string, +) error { + if t.taskStore == nil { + return nil + } + delegationID := parseUUIDOrNil(req.DelegationID) + tenantID := parseUUIDOrNil(req.TenantID) + if delegationID == uuid.Nil || tenantID == uuid.Nil || req.FromAgentID == uuid.Nil { + return fmt.Errorf("delegation completion requires valid tenant, source agent, and delegation ID") + } + dbCtx := store.WithTenantID(context.Background(), tenantID) + if err := retryTerminalPersistence(dbCtx, func(attemptCtx context.Context) error { + return t.taskStore.UpdateStatus( + attemptCtx, + req.FromAgentID, + delegationID, + status, + result, + 0, + 0, + 0, + ) + }); err != nil { + slog.Error("delegate.async.completion_persist_failed", + "delegation_id", req.DelegationID, + "from_agent_id", req.FromAgentID, + "status", status, + "error", err, + ) + return err + } + return nil +} + +// updateDelegateRunning is observability-only: execution may proceed even when +// this transition cannot be persisted. Keep it to one short attempt outside +// the admission callback so a database outage cannot monopolize a child-run +// permit. +func (t *DelegateTool) updateDelegateRunning(req DelegateRequest) error { + if t.taskStore == nil { + return nil + } + delegationID := parseUUIDOrNil(req.DelegationID) + tenantID := parseUUIDOrNil(req.TenantID) + if delegationID == uuid.Nil || tenantID == uuid.Nil || req.FromAgentID == uuid.Nil { + return fmt.Errorf("delegation completion requires valid tenant, source agent, and delegation ID") + } + dbCtx, cancel := context.WithTimeout( + store.WithTenantID(context.Background(), tenantID), + delegateRunningPersistenceTimeout, + ) + defer cancel() + err := t.taskStore.UpdateStatus( + dbCtx, + req.FromAgentID, + delegationID, + TaskStatusRunning, + nil, + 0, + 0, + 0, + ) + if err != nil { + slog.Warn("delegate.async.running_persist_failed", + "delegation_id", req.DelegationID, + "from_agent_id", req.FromAgentID, + "error", err, + ) + } + return err +} + +func (t *DelegateTool) updateDelegateCompletionMedia( + req DelegateRequest, + media []persistedCompletionMedia, +) error { + if t.taskStore == nil || len(media) == 0 { + return nil + } + delegationID := parseUUIDOrNil(req.DelegationID) + tenantID := parseUUIDOrNil(req.TenantID) + if delegationID == uuid.Nil || tenantID == uuid.Nil || req.FromAgentID == uuid.Nil { + return fmt.Errorf("delegation completion requires valid tenant, source agent, and delegation ID") + } + dbCtx := store.WithTenantID(context.Background(), tenantID) + if err := retryTerminalPersistence(dbCtx, func(attemptCtx context.Context) error { + return t.taskStore.UpdateMetadata(attemptCtx, req.FromAgentID, delegationID, map[string]any{ + asyncCompletionMediaKey: media, + }) + }); err != nil { + slog.Error("delegate.async.completion_media_persist_failed", + "delegation_id", req.DelegationID, + "from_agent_id", req.FromAgentID, + "error", err, + ) + return err + } + return nil +} + +func (t *DelegateTool) updateDelegateAnnouncement(req DelegateRequest, delivered bool) { + if t.taskStore == nil { + return + } + delegationID := parseUUIDOrNil(req.DelegationID) + tenantID := parseUUIDOrNil(req.TenantID) + if delegationID == uuid.Nil || tenantID == uuid.Nil || req.FromAgentID == uuid.Nil { + return + } + status := asyncCompletionDeliveryMissed + if delivered { + status = asyncCompletionDeliveryDone + } + dbCtx := store.WithTenantID(context.Background(), tenantID) + if err := retryAsyncPersistence(dbCtx, func(attemptCtx context.Context) error { + return t.taskStore.UpdateMetadata(attemptCtx, req.FromAgentID, delegationID, map[string]any{ + asyncCompletionDeliveryKey: status, + }) + }); err != nil { + slog.Warn("delegate.async.announcement_status_failed", + "delegation_id", req.DelegationID, + "error", err, + ) + } +} + +func (t *DelegateTool) executeGetCompletion(ctx context.Context, args map[string]any) *Result { + rawID, _ := args["delegation_id"].(string) + delegationID, err := uuid.Parse(rawID) + if err != nil { + return ErrorResult("delegation_id must be a valid UUID") + } + tenantID := store.TenantIDFromContext(ctx) + fromAgentID := store.AgentIDFromContext(ctx) + if tenantID == uuid.Nil || fromAgentID == uuid.Nil { + return ErrorResult("delegate result lookup requires tenant and agent context") + } + if t.taskStore == nil { + return ErrorResult("durable delegation task tracking is unavailable") + } + dbCtx := store.WithTenantID(context.WithoutCancel(ctx), tenantID) + task, err := t.taskStore.Get(dbCtx, fromAgentID, delegationID) + if err != nil { + return ErrorResult("failed to load delegation result") + } + if task == nil || completionKind(task.Metadata) != asyncCompletionKindDelegate { + return ErrorResult("delegation result not found") + } + payload := persistedCompletionPayload(task) + payload["delegation_id"] = task.ID.String() + delete(payload, "completion_id") + if target, ok := task.Metadata[delegateCompletionTargetKey].(string); ok && target != "" { + payload["agent"] = target + } + encoded, err := json.Marshal(payload) + if err != nil { + return ErrorResult("failed to encode delegation result") + } + return NewResult(string(encoded)) +} diff --git a/internal/tools/delegate_tool.go b/internal/tools/delegate_tool.go index 670aa185..96e86cea 100644 --- a/internal/tools/delegate_tool.go +++ b/internal/tools/delegate_tool.go @@ -3,22 +3,37 @@ package tools import ( "context" "encoding/json" + "errors" "fmt" "log/slog" + "path/filepath" + "slices" + "strings" + "sync" "time" "github.com/google/uuid" "github.com/nextlevelbuilder/goclaw/internal/bus" + orchestration "github.com/nextlevelbuilder/goclaw/internal/childrun" + "github.com/nextlevelbuilder/goclaw/internal/config" "github.com/nextlevelbuilder/goclaw/internal/eventbus" "github.com/nextlevelbuilder/goclaw/internal/hooks" "github.com/nextlevelbuilder/goclaw/internal/store" + "github.com/nextlevelbuilder/goclaw/internal/tracing" +) + +const ( + delegationArtifactFailureTTL = 60 * time.Minute + delegationArtifactSweepInterval = time.Minute + delegationArtifactSweepBatch = 32 ) // DelegateResult carries the delegatee's response content and any media produced. type DelegateResult struct { Content string Media []bus.MediaFile + TraceID uuid.UUID } // DelegateRunFunc dispatches a delegation to a target agent. @@ -28,19 +43,25 @@ type DelegateRunFunc func(ctx context.Context, req DelegateRequest) (DelegateRes // DelegateRequest describes a delegation dispatch. type DelegateRequest struct { - FromAgentID uuid.UUID - FromAgentKey string - ToAgentKey string - Task string - DelegationID string - UserID string - SenderID string // real acting sender preserved through delegate announce re-ingress (#915) - Role string // caller's RBAC role; bypasses per-user grants for admin/operator/owner (#915) - TenantID string - Channel string - ChatID string - PeerKind string - SessionKey string + FromAgentID uuid.UUID + FromAgentKey string + ToAgentKey string + Task string + DelegateInputsPath string + DelegateOutputsPath string + DelegationID string + UserID string + SenderID string // real acting sender preserved through delegate announce re-ingress (#915) + Role string // caller's RBAC role; bypasses per-user grants for admin/operator/owner (#915) + TenantID string + Channel string + ChannelType string + ChatID string + PeerKind string + SessionKey string + OriginTraceID uuid.UUID + OriginRootSpanID uuid.UUID + OnTraceCreated func(uuid.UUID) } // DelegateTool implements the `delegate` tool for inter-agent task delegation. @@ -50,19 +71,135 @@ type DelegateTool struct { agents store.AgentCRUDStore eventBus eventbus.DomainEventBus runFn DelegateRunFunc - msgBus *bus.MessageBus // for async announce back to parent - hookDispatcher hooks.Dispatcher // optional; nil-safe + msgBus *bus.MessageBus // for async announce back to parent + taskStore store.SubagentTaskStore // durable async completion ledger + hookDispatcher hooks.Dispatcher // optional; nil-safe + admission *orchestration.ChildRunAdmission + workspace string + dataDir string + removeExchange func(string, uuid.UUID) error + + retainedMu sync.Mutex + retained map[string]retainedDelegationArtifact + sweeperStarted bool + sweeperClosed bool + sweeperStop chan struct{} + sweeperDone chan struct{} + + completionMu sync.Mutex + completionClosed bool + completionWG sync.WaitGroup + closeOnce sync.Once + closeDone chan struct{} } // SetMsgBus sets the message bus for async result delivery to parent agent. func (t *DelegateTool) SetMsgBus(mb *bus.MessageBus) { t.msgBus = mb } +// SetTaskStore configures the durable ledger used by async delegations. +func (t *DelegateTool) SetTaskStore(s store.SubagentTaskStore) { t.taskStore = s } + // SetHookDispatcher sets the hook dispatcher for SubagentStart/Stop events. func (t *DelegateTool) SetHookDispatcher(d hooks.Dispatcher) { t.hookDispatcher = d } +// SetWorkspace sets the global managed workspace used to derive canonical +// tenant-scoped delegation exchanges. The caller's effective workspace is +// captured separately from the tool context for every dispatch. +func (t *DelegateTool) SetWorkspace(workspace string) { + t.workspace = workspace + t.recoverRetainedDelegationExchanges() +} + +// SetDataDir supplies the second managed root used by Team workspaces. It is +// needed only to retry publication-temp cleanup after restart. +func (t *DelegateTool) SetDataDir(dataDir string) { t.dataDir = dataDir } + +// Close stops the failed-exchange retention sweeper and drains accepted async +// completion persistence/announcement work. +func (t *DelegateTool) Close() { + _ = t.CloseContext(context.Background()) +} + +// CloseContext prevents new async completion ownership and waits for accepted +// completion work. Gateway callers close child-run admission before this drain. +func (t *DelegateTool) CloseContext(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() + } + t.closeOnce.Do(func() { + t.completionMu.Lock() + t.completionClosed = true + t.completionMu.Unlock() + + t.retainedMu.Lock() + t.sweeperClosed = true + sweeperStarted := t.sweeperStarted + if sweeperStarted { + close(t.sweeperStop) + } + t.retainedMu.Unlock() + + go func() { + if sweeperStarted { + <-t.sweeperDone + } + t.completionWG.Wait() + close(t.closeDone) + }() + }) + select { + case <-t.closeDone: + return nil + case <-ctx.Done(): + return fmt.Errorf("delegate completion drain timeout: %w", ctx.Err()) + } +} + +func (t *DelegateTool) beginAsyncCompletion() (func(), bool) { + t.completionMu.Lock() + defer t.completionMu.Unlock() + if t.completionClosed { + return nil, false + } + t.completionWG.Add(1) + var once sync.Once + return func() { + once.Do(t.completionWG.Done) + }, true +} + // NewDelegateTool creates a delegate tool. func NewDelegateTool(links store.AgentLinkStore, agents store.AgentCRUDStore, eb eventbus.DomainEventBus, runFn DelegateRunFunc) *DelegateTool { - return &DelegateTool{links: links, agents: agents, eventBus: eb, runFn: runFn} + return NewDelegateToolWithAdmission( + links, + agents, + eb, + runFn, + orchestration.NewChildRunAdmission(32, 128), + ) +} + +func NewDelegateToolWithAdmission( + links store.AgentLinkStore, + agents store.AgentCRUDStore, + eb eventbus.DomainEventBus, + runFn DelegateRunFunc, + admission *orchestration.ChildRunAdmission, +) *DelegateTool { + if admission == nil { + admission = orchestration.NewChildRunAdmission(32, 128) + } + return &DelegateTool{ + links: links, + agents: agents, + eventBus: eb, + runFn: runFn, + admission: admission, + retained: make(map[string]retainedDelegationArtifact), + sweeperStop: make(chan struct{}), + sweeperDone: make(chan struct{}), + closeDone: make(chan struct{}), + } } func (t *DelegateTool) Name() string { return "delegate" } @@ -79,6 +216,15 @@ func (t *DelegateTool) Parameters() map[string]any { "type": "string", "description": "The agent_key of the target agent to delegate to", }, + "action": map[string]any{ + "type": "string", + "enum": []string{"delegate", "get"}, + "description": "delegate (default) starts work; get retrieves a durable async result", + }, + "delegation_id": map[string]any{ + "type": "string", + "description": "Delegation UUID returned by async mode (required for action=get)", + }, "task": map[string]any{ "type": "string", "description": "Description of the task to delegate", @@ -92,12 +238,27 @@ func (t *DelegateTool) Parameters() map[string]any { "type": "integer", "description": "Timeout in seconds for sync mode (default: 300)", }, + "inputs": map[string]any{ + "type": "array", + "items": map[string]any{"type": "string"}, + "maxItems": DelegationArtifactMaxFiles, + "description": "Caller-workspace relative files to stage as read-only delegation inputs", + }, }, - "required": []string{"agent_key", "task"}, } } func (t *DelegateTool) Execute(ctx context.Context, args map[string]any) *Result { + action, _ := args["action"].(string) + if action == "" { + action = "delegate" + } + if action == "get" { + return t.executeGetCompletion(ctx, args) + } + if action != "delegate" { + return ErrorResult(fmt.Sprintf("unknown delegate action %q", action)) + } agentKey, _ := args["agent_key"].(string) task, _ := args["task"].(string) mode, _ := args["mode"].(string) @@ -115,6 +276,13 @@ func (t *DelegateTool) Execute(ctx context.Context, args map[string]any) *Result if agentKey == "" || task == "" { return ErrorResult("agent_key and task are required") } + if err := validateDelegationChildRunMode(ctx, "delegate", mode); err != nil { + return ErrorResult(err.Error()) + } + explicitInputs, err := parseDelegateInputs(args["inputs"]) + if err != nil { + return ErrorResult(err.Error()) + } // Resolve calling agent from context fromAgentID := store.AgentIDFromContext(ctx) @@ -128,20 +296,42 @@ func (t *DelegateTool) Execute(ctx context.Context, args map[string]any) *Result return ErrorResult(fmt.Sprintf("target agent %q not found", agentKey)) } - // Permission check via agent_links - allowed, err := t.links.CanDelegate(ctx, fromAgentID, target.ID) + // Resolve the effective directional link for permission only. + // max_concurrent is retained as compatibility metadata and is not enforced. + link, err := t.links.GetLinkBetween(ctx, fromAgentID, target.ID) if err != nil { slog.Warn("delegate.permission_check_error", "from", fromAgentID, "to", target.ID, "error", err) return ErrorResult("failed to check delegation permission") } - if !allowed { + if link == nil { return ErrorResult(fmt.Sprintf("no delegation link from current agent to %q", agentKey)) } + _ = link - delegationID := uuid.New().String() - // Audit-trail identity = actor (real sender). Groups audit actions to the - // individual user rather than the group principal (#915). - actorID := store.ActorIDFromContext(ctx) + tenantID := store.TenantIDFromContext(ctx) + if tenantID == uuid.Nil || t.workspace == "" { + return ErrorResult("delegation artifact workspace is unavailable") + } + callerWorkspace := ToolWorkspaceFromCtx(ctx) + if callerWorkspace == "" { + return ErrorResult("delegation caller workspace is unavailable") + } + callerWorkspace, err = filepath.Abs(callerWorkspace) + if err != nil { + return ErrorResult("delegation caller workspace is unavailable") + } + callerRoot, err := OpenDelegationArtifactRoot(callerWorkspace) + if err != nil { + return ErrorResult(err.Error()) + } + inputs, err := collectDelegateInputs(callerWorkspace, explicitInputs, RunMediaPathsFromCtx(ctx)) + if err != nil { + _ = callerRoot.Close() + return ErrorResult(err.Error()) + } + + delegationUUID := uuid.New() + delegationID := delegationUUID.String() req := DelegateRequest{ FromAgentID: fromAgentID, @@ -149,80 +339,86 @@ func (t *DelegateTool) Execute(ctx context.Context, args map[string]any) *Result ToAgentKey: agentKey, Task: task, DelegationID: delegationID, - UserID: actorID, - SenderID: store.SenderIDFromContext(ctx), - Role: store.RoleFromContext(ctx), - TenantID: store.TenantIDFromContext(ctx).String(), - Channel: ToolChannelFromCtx(ctx), - ChatID: ToolChatIDFromCtx(ctx), - PeerKind: ToolPeerKindFromCtx(ctx), - SessionKey: ToolSessionKeyFromCtx(ctx), + // Preserve the authorization scope separately from the acting sender. + // Group file permissions require both the group principal (UserID) and + // the real individual (SenderID), matching teammate task dispatch. + UserID: store.UserIDFromContext(ctx), + SenderID: store.SenderIDFromContext(ctx), + Role: store.RoleFromContext(ctx), + TenantID: tenantID.String(), + Channel: ToolChannelFromCtx(ctx), + ChannelType: ToolChannelTypeFromCtx(ctx), + ChatID: ToolChatIDFromCtx(ctx), + PeerKind: ToolPeerKindFromCtx(ctx), + SessionKey: ToolSessionKeyFromCtx(ctx), + OriginTraceID: tracing.TraceIDFromContext(ctx), + OriginRootSpanID: tracing.ParentSpanIDFromContext(ctx), } - - // Emit delegate.sent event - t.emitEvent(ctx, eventbus.EventDelegateSent, eventbus.DelegateSentPayload{ - DelegationID: delegationID, - FromAgent: req.FromAgentKey, - ToAgent: agentKey, - Task: task, - Mode: mode, - }) - - // Fire SubagentStart hook (blocking). Nil-safe: skip if no dispatcher. - if t.hookDispatcher != nil { - evt := hooks.Event{ - EventID: uuid.NewString(), - SessionID: req.SessionKey, - TenantID: parseUUIDOrNil(req.TenantID), - AgentID: req.FromAgentID, - HookEvent: hooks.EventSubagentStart, - Depth: hooks.DepthFrom(ctx), - } - r, err := t.hookDispatcher.Fire(ctx, evt) - if err != nil { - t.emitEvent(ctx, eventbus.EventDelegateFailed, eventbus.DelegateFailedPayload{ - DelegationID: req.DelegationID, - FromAgent: req.FromAgentKey, - ToAgent: req.ToAgentKey, - Error: fmt.Sprintf("subagent_start hook error: %v", err), - }) - return ErrorResult(fmt.Sprintf("subagent_start hook error: %v", err)) - } - // Updated* from FireResult intentionally unused — delegate has no - // mutation need in Wave 1. - if r.Decision == hooks.DecisionBlock { - t.emitEvent(ctx, eventbus.EventDelegateFailed, eventbus.DelegateFailedPayload{ - DelegationID: req.DelegationID, - FromAgent: req.FromAgentKey, - ToAgent: req.ToAgentKey, - Error: "blocked by subagent_start hook", - }) - return ErrorResult(fmt.Sprintf("delegation to %q blocked by hook policy", req.ToAgentKey)) - } - // Increment depth so nested delegate calls honor MaxLoopDepth. - ctx = hooks.IncDepth(ctx) + job := &delegateArtifactJob{ + req: req, + mode: mode, + callerRoot: callerRoot, + callerWorkspace: callerWorkspace, + tenantWorkspace: config.TenantWorkspace(t.workspace, tenantID, store.TenantSlugFromContext(ctx)), + tenantID: tenantID, + tenantSlug: store.TenantSlugFromContext(ctx), + delegationID: delegationUUID, + inputs: inputs, } + job.callerLocation = t.resolveDelegationCallerLocation(job) if mode == "sync" { - return t.executeSyncMode(ctx, req, timeoutSec) + return t.executeSyncMode(ctx, job, timeoutSec) } - return t.executeAsyncMode(ctx, req) + return t.executeAsyncMode(ctx, job) } // executeSyncMode blocks until the delegatee completes or timeout. -func (t *DelegateTool) executeSyncMode(ctx context.Context, req DelegateRequest, timeoutSec int) *Result { +func (t *DelegateTool) executeSyncMode(ctx context.Context, job *delegateArtifactJob, timeoutSec int) *Result { + req := job.req syncCtx, cancel := context.WithTimeout(ctx, time.Duration(timeoutSec)*time.Second) defer cancel() - dr, err := t.runFn(syncCtx, req) - if err != nil { + var dr DelegateResult + var runErr error + constraints := delegateAdmissionConstraints(ctx, req) + run := func(runCtx context.Context, lease *orchestration.ChildRunLease) { + defer job.closeCallerRoot() + runCtx = withDelegatedAgentExecution(runCtx, lease) + dr, runErr = t.runArtifactExchange(runCtx, job) + lease.Release() + } + if parentLease := childRunLeaseFromContext(ctx); parentLease != nil { + continueErr := parentLease.Continue(syncCtx, constraints, run) + if continueErr != nil { + job.closeCallerRoot() + runErr = continueErr + } + } else { + ticket, err := t.admission.Enqueue(syncCtx, constraints, run) + if err != nil { + job.closeCallerRoot() + return ErrorResult(err.Error()) + } + if err := ticket.Activate(); err != nil { + ticket.Cancel() + job.closeCallerRoot() + return ErrorResult(err.Error()) + } + <-ticket.Done() + job.closeCallerRoot() + if runErr == nil { + runErr = ticket.Err() + } + } + if runErr != nil { t.emitEvent(ctx, eventbus.EventDelegateFailed, eventbus.DelegateFailedPayload{ DelegationID: req.DelegationID, FromAgent: req.FromAgentKey, ToAgent: req.ToAgentKey, - Error: err.Error(), + Error: runErr.Error(), }) - return ErrorResult(fmt.Sprintf("delegation to %q failed: %v", req.ToAgentKey, err)) + return ErrorResult(fmt.Sprintf("delegation to %q failed: %v", req.ToAgentKey, runErr)) } t.emitEvent(ctx, eventbus.EventDelegateCompleted, eventbus.DelegateCompletedPayload{ @@ -245,33 +441,139 @@ func (t *DelegateTool) executeSyncMode(ctx context.Context, req DelegateRequest, } // executeAsyncMode spawns a goroutine and returns immediately. -func (t *DelegateTool) executeAsyncMode(ctx context.Context, req DelegateRequest) *Result { - // Detach from parent cancel but add a deadline to prevent goroutine leaks. +func (t *DelegateTool) executeAsyncMode(ctx context.Context, job *delegateArtifactJob) *Result { + req := job.req + finishCompletion, ok := t.beginAsyncCompletion() + if !ok { + job.closeCallerRoot() + return ErrorResult("delegate tool is closing") + } + completionTransferred := false + defer func() { + if !completionTransferred { + finishCompletion() + } + }() + // Detach from parent cancellation but keep a bounded admitted callback. bgCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Minute) + announceCtx := context.WithoutCancel(ctx) + var dr DelegateResult + var runErr error + runStarted := make(chan struct{}) + ticket, err := t.admission.Enqueue(bgCtx, delegateAdmissionConstraints(ctx, req), func(runCtx context.Context, lease *orchestration.ChildRunLease) { + defer job.closeCallerRoot() + close(runStarted) + runCtx = withDelegatedAgentExecution(runCtx, lease) + dr, runErr = t.runArtifactExchange(runCtx, job) + lease.Release() + }) + if err != nil { + cancel() + job.closeCallerRoot() + return ErrorResult(err.Error()) + } + if err := t.createDelegateCompletion(ctx, req); err != nil { + ticket.Cancel() + cancel() + job.closeCallerRoot() + return ErrorResult(err.Error()) + } + if err := ticket.Activate(); err != nil { + ticket.Cancel() + cancel() + job.closeCallerRoot() + result := err.Error() + _ = t.updateDelegateCompletion(req, TaskStatusFailed, &result) + return ErrorResult(err.Error()) + } + // Nonterminal observability persistence runs outside the admitted callback. + // Terminal persistence and announcements wait for the callback to return, + // which proves its execution permit has been released. go func() { - defer cancel() - dr, err := t.runFn(bgCtx, req) - if err != nil { - t.emitEvent(bgCtx, eventbus.EventDelegateFailed, eventbus.DelegateFailedPayload{ + defer finishCompletion() + started := false + select { + case <-runStarted: + started = true + case <-ticket.Done(): + select { + case <-runStarted: + started = true + default: + } + } + if started { + _ = t.updateDelegateRunning(req) + } + <-ticket.Done() + job.closeCallerRoot() + cancel() + if runErr == nil { + runErr = ticket.Err() + } + if runErr != nil { + result := runErr.Error() + terminalPersisted := t.updateDelegateCompletion(req, TaskStatusFailed, &result) == nil + t.emitEvent(announceCtx, eventbus.EventDelegateFailed, eventbus.DelegateFailedPayload{ DelegationID: req.DelegationID, FromAgent: req.FromAgentKey, ToAgent: req.ToAgentKey, - Error: err.Error(), + Error: runErr.Error(), }) - slog.Warn("delegate.async.failed", "to", req.ToAgentKey, "error", err) - t.announceToParent(req, fmt.Sprintf("[Delegation to %s failed: %v]", req.ToAgentKey, err), nil) + slog.Warn("delegate.async.failed", "to", req.ToAgentKey, "error", runErr) + content := fmt.Sprintf("[Delegation to %s failed: %v]", req.ToAgentKey, runErr) + delivered := t.announceToParent(req, content, nil) + if terminalPersisted { + t.updateDelegateAnnouncement(req, delivered) + } else { + slog.Error("delegate.async.announce_without_durable_terminal", + "delegation_id", req.DelegationID, + "to", req.ToAgentKey, + "delivered", delivered, + ) + } + if !delivered { + slog.Warn("delegate.async.announce_deferred_to_ledger", + "delegation_id", req.DelegationID, + "to", req.ToAgentKey, + "reason", "inbound_bus_full", + ) + } return } - t.emitEvent(bgCtx, eventbus.EventDelegateCompleted, eventbus.DelegateCompletedPayload{ + completionMedia := completionMediaDescriptors(dr.Media, job.callerWorkspace, "") + terminalPersisted := t.updateDelegateCompletionMedia(req, completionMedia) == nil + if terminalPersisted { + terminalPersisted = t.updateDelegateCompletion(req, TaskStatusCompleted, &dr.Content) == nil + } + t.emitEvent(announceCtx, eventbus.EventDelegateCompleted, eventbus.DelegateCompletedPayload{ DelegationID: req.DelegationID, FromAgent: req.FromAgentKey, ToAgent: req.ToAgentKey, Content: truncate(dr.Content, 500), MediaCount: len(dr.Media), }) - t.announceToParent(req, fmt.Sprintf("[Delegation result from %s]\n\n%s", req.ToAgentKey, dr.Content), dr.Media) + content := fmt.Sprintf("[Delegation result from %s]\n\n%s", req.ToAgentKey, dr.Content) + delivered := t.announceToParent(req, content, dr.Media) + if terminalPersisted { + t.updateDelegateAnnouncement(req, delivered) + } else { + slog.Error("delegate.async.announce_without_durable_terminal", + "delegation_id", req.DelegationID, + "to", req.ToAgentKey, + "delivered", delivered, + ) + } + if !delivered { + slog.Warn("delegate.async.announce_deferred_to_ledger", + "delegation_id", req.DelegationID, + "to", req.ToAgentKey, + "reason", "inbound_bus_full", + ) + } }() + completionTransferred = true result, _ := json.Marshal(map[string]any{ "delegation_id": req.DelegationID, @@ -282,11 +584,567 @@ func (t *DelegateTool) executeAsyncMode(ctx context.Context, req DelegateRequest return NewResult(string(result)) } +func (t *DelegateTool) runAdmitted( + ctx context.Context, + req DelegateRequest, + onDispatch func(context.Context), +) (DelegateResult, error) { + if t.hookDispatcher != nil { + evt := hooks.Event{ + EventID: uuid.NewString(), + SessionID: req.SessionKey, + TenantID: parseUUIDOrNil(req.TenantID), + AgentID: req.FromAgentID, + HookEvent: hooks.EventSubagentStart, + Depth: hooks.DepthFrom(ctx), + } + result, err := t.hookDispatcher.Fire(ctx, evt) + if err != nil { + return DelegateResult{}, fmt.Errorf("subagent_start hook error: %w", err) + } + if result.Decision == hooks.DecisionBlock { + return DelegateResult{}, fmt.Errorf("delegation to %q blocked by hook policy", req.ToAgentKey) + } + ctx = hooks.IncDepth(ctx) + } + if onDispatch != nil { + onDispatch(ctx) + } + return t.runFn(ctx, req) +} + +type delegateInput struct { + relativePath string + taskReferences []string +} + +type delegateArtifactJob struct { + req DelegateRequest + mode string + callerRoot *DelegationArtifactRoot + callerWorkspace string + tenantWorkspace string + tenantID uuid.UUID + tenantSlug string + delegationID uuid.UUID + inputs []delegateInput + callerLocation *delegationArtifactCallerLocation + closeOnce sync.Once +} + +func (j *delegateArtifactJob) closeCallerRoot() { + j.closeOnce.Do(func() { + _ = j.callerRoot.Close() + }) +} + +func parseDelegateInputs(raw any) ([]string, error) { + if raw == nil { + return nil, nil + } + values, ok := raw.([]any) + if !ok { + if typed, typedOK := raw.([]string); typedOK { + values = make([]any, len(typed)) + for i, value := range typed { + values[i] = value + } + } else { + return nil, fmt.Errorf("inputs must be an array of relative paths") + } + } + if len(values) > DelegationArtifactMaxFiles { + return nil, fmt.Errorf("inputs may contain at most %d paths", DelegationArtifactMaxFiles) + } + inputs := make([]string, 0, len(values)) + for i, value := range values { + rawPath, ok := value.(string) + if !ok { + return nil, fmt.Errorf("input %d must be a relative path", i+1) + } + normalized, err := validateArtifactRelativePath(rawPath) + if err != nil { + return nil, fmt.Errorf("input %d: %w", i+1, err) + } + inputs = append(inputs, normalized) + } + return inputs, nil +} + +func collectDelegateInputs(callerWorkspace string, explicitInputs, currentMedia []string) ([]delegateInput, error) { + inputs := make([]delegateInput, 0, len(explicitInputs)+len(currentMedia)) + positions := make(map[string]int, cap(inputs)) + add := func(relativePath string, taskReferences ...string) { + if position, exists := positions[relativePath]; exists { + for _, reference := range taskReferences { + if reference != "" && !slices.Contains(inputs[position].taskReferences, reference) { + inputs[position].taskReferences = append(inputs[position].taskReferences, reference) + } + } + return + } + positions[relativePath] = len(inputs) + inputs = append(inputs, delegateInput{ + relativePath: relativePath, + taskReferences: slices.DeleteFunc(taskReferences, func(reference string) bool { + return reference == "" + }), + }) + } + for _, relativePath := range explicitInputs { + add(relativePath) + } + for i, mediaPath := range currentMedia { + absolutePath, err := filepath.Abs(mediaPath) + if err != nil { + return nil, fmt.Errorf("current media input %d is unavailable", i+1) + } + relativePath, err := filepath.Rel(callerWorkspace, absolutePath) + if err != nil { + return nil, fmt.Errorf("current media input %d is outside the caller workspace", i+1) + } + relativePath = filepath.ToSlash(relativePath) + normalized, err := validateArtifactRelativePath(relativePath) + if err != nil || normalized == "." || strings.HasPrefix(normalized, "../") { + return nil, fmt.Errorf("current media input %d is outside the caller workspace", i+1) + } + add(normalized, absolutePath, filepath.ToSlash(absolutePath)) + } + if len(inputs) > DelegationArtifactMaxFiles { + return nil, fmt.Errorf("delegation inputs may contain at most %d files", DelegationArtifactMaxFiles) + } + return inputs, nil +} + +func (t *DelegateTool) runArtifactExchange(ctx context.Context, job *delegateArtifactJob) (dr DelegateResult, returnErr error) { + exchange, err := NewDelegationArtifactExchange( + job.tenantWorkspace, + job.tenantID, + job.delegationID, + DelegationArtifactLimits{}, + delegationArtifactFailureTTL, + ) + if err != nil { + return DelegateResult{}, err + } + durablePublished := false + publicationTempPath := "" + var traceID uuid.UUID + var staged []DelegationArtifact + var publication DelegationArtifactPublication + var stagedTraceOnce sync.Once + emitStaged := func(id uuid.UUID, at time.Time) { + if id == uuid.Nil { + return + } + stagedTraceOnce.Do(func() { + traceID = id + emitDelegationArtifactLifecycleSpan( + ctx, + job, + id, + "staged", + at, + traceOutputsFromStaged(staged), + ) + }) + } + defer func() { + if !durablePublished { + exchange.RetainFailure(time.Now(), artifactErrorCode(returnErr)) + lifecycleStatus := artifactLifecycleFailed + traceStatus := "failed" + if errors.Is(returnErr, context.Canceled) || + errors.Is(returnErr, context.DeadlineExceeded) { + lifecycleStatus = artifactLifecycleCancelled + traceStatus = "cancelled" + } + if err := t.registerRetainedDelegationExchange(exchange, job, lifecycleStatus); err != nil { + returnErr = errors.Join(returnErr, err) + } + if traceID == uuid.Nil { + traceID = job.req.OriginTraceID + } + emitDelegationArtifactLifecycleSpan( + ctx, + job, + traceID, + traceStatus, + time.Now(), + traceOutputsFromStaged(staged), + ) + } + if err := exchange.Close(); err != nil { + slog.Warn("delegate.artifact_exchange_close_failed", "delegation_id", job.req.DelegationID) + } + if durablePublished { + if err := t.tryRemoveDelegationExchange(job.tenantWorkspace, job.delegationID); err != nil { + slog.Warn("delegate.artifact_exchange_cleanup_failed", "delegation_id", job.req.DelegationID) + cleanupCtx := context.WithoutCancel(ctx) + t.registerPublishedDelegationCleanup(job, publicationTempPath, func() { + emitDelegationArtifactLifecycleSpan( + cleanupCtx, + job, + traceID, + "cleaned", + time.Now(), + traceOutputsFromManifest(publication.Manifest), + ) + }) + } else { + emitDelegationArtifactLifecycleSpan( + ctx, + job, + traceID, + "cleaned", + time.Now(), + traceOutputsFromManifest(publication.Manifest), + ) + } + } + }() + if err := t.updateActiveDelegationLifecycle(exchange, job); err != nil { + return DelegateResult{}, err + } + + relativePaths := make([]string, len(job.inputs)) + for i, input := range job.inputs { + relativePaths[i] = input.relativePath + } + stagedAt := time.Now() + staged, err = exchange.StageInputs(ctx, job.callerRoot, relativePaths) + if err != nil { + return DelegateResult{}, err + } + if err := t.markDelegationRunning(exchange, job, time.Now()); err != nil { + return DelegateResult{}, err + } + + req := job.req + req.Task = delegationTaskWithInputAliases(req.Task, job.inputs, staged) + req.DelegateInputsPath = exchange.InputsMount().HostRoot + req.DelegateOutputsPath = exchange.OutputsHostPath() + req.OnTraceCreated = func(id uuid.UUID) { + emitStaged(id, stagedAt) + } + dr, err = t.runAdmitted(ctx, req, func(dispatchCtx context.Context) { + t.emitEvent(dispatchCtx, eventbus.EventDelegateSent, eventbus.DelegateSentPayload{ + DelegationID: req.DelegationID, + FromAgent: req.FromAgentKey, + ToAgent: req.ToAgentKey, + Task: delegationEventTask(req.Task, job.callerWorkspace, exchange), + Mode: job.mode, + }) + }) + if dr.TraceID != uuid.Nil { + traceID = dr.TraceID + emitStaged(dr.TraceID, stagedAt) + } else if traceID == uuid.Nil { + traceID = job.req.OriginTraceID + emitStaged(traceID, stagedAt) + } + if err != nil { + return DelegateResult{}, redactDelegationArtifactError(err, exchange) + } + dr.Content = redactDelegationArtifactText(dr.Content, exchange) + dr.Media = nil + + publication, err = exchange.publishWithPreparation( + ctx, + job.callerRoot, + time.Now(), + func(tempPath string) error { + publicationTempPath = tempPath + return t.markDelegationPublishing(exchange, job, tempPath, time.Now()) + }, + ) + if err != nil { + return DelegateResult{}, err + } + durablePublished = true + if err := t.markDelegationPublished(exchange, job, publication.Manifest.PublishedAt); err != nil { + // The no-replace rename and directory sync completed before this + // best-effort lifecycle update. Reporting the delegation as failed here + // invites a retry that creates a second durable publication even though + // the caller already owns the first one. + slog.Warn("delegate.artifact_lifecycle_published_failed", + "delegation_id", job.req.DelegationID, + "error", err, + ) + } + dr.Media = publicationMedia(job.callerWorkspace, publication) + emitDelegationArtifactLifecycleSpan( + ctx, + job, + traceID, + "published", + publication.Manifest.PublishedAt, + traceOutputsFromManifest(publication.Manifest), + ) + return dr, nil +} + +type delegationArtifactTraceOutput struct { + Path string `json:"path"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + MediaType string `json:"media_type"` +} + +type delegationArtifactTraceEvent struct { + DelegationID string `json:"delegation_id"` + OccurredAt time.Time `json:"occurred_at"` + ArtifactCount int `json:"artifact_count"` + ArtifactBytes int64 `json:"artifact_bytes"` + Artifacts []delegationArtifactTraceOutput `json:"artifacts"` + Status string `json:"status"` +} + +func emitDelegationArtifactLifecycleSpan( + ctx context.Context, + job *delegateArtifactJob, + traceID uuid.UUID, + status string, + occurredAt time.Time, + artifacts []delegationArtifactTraceOutput, +) { + collector := tracing.CollectorFromContext(ctx) + if collector == nil || traceID == uuid.Nil { + return + } + var artifactBytes int64 + for _, artifact := range artifacts { + artifactBytes += artifact.SizeBytes + } + event := delegationArtifactTraceEvent{ + DelegationID: job.delegationID.String(), + OccurredAt: occurredAt.UTC(), + ArtifactCount: len(artifacts), + ArtifactBytes: artifactBytes, + Artifacts: artifacts, + Status: status, + } + metadata, err := json.Marshal(event) + if err != nil { + return + } + now := time.Now().UTC() + collector.EmitSpan(tracing.RedactSpan(ctx, store.SpanData{ + ID: store.GenNewID(), + TraceID: traceID, + SpanType: store.SpanTypeEvent, + Name: "delegate.artifacts." + status, + StartTime: now, + EndTime: &now, + Status: store.SpanStatusCompleted, + Level: store.SpanLevelDefault, + Metadata: metadata, + TenantID: job.tenantID, + CreatedAt: now, + })) +} + +func traceOutputsFromStaged( + staged []DelegationArtifact, +) []delegationArtifactTraceOutput { + outputs := make([]delegationArtifactTraceOutput, len(staged)) + for i, artifact := range staged { + outputs[i] = delegationArtifactTraceOutput{ + Path: artifact.Path, + SizeBytes: artifact.SizeBytes, + SHA256: artifact.SHA256, + MediaType: artifact.MediaType, + } + } + return outputs +} + +func traceOutputsFromManifest( + manifest DelegationArtifactManifest, +) []delegationArtifactTraceOutput { + outputs := make([]delegationArtifactTraceOutput, len(manifest.Outputs)) + for i, output := range manifest.Outputs { + outputs[i] = delegationArtifactTraceOutput{ + Path: output.Path, + SizeBytes: output.SizeBytes, + SHA256: output.SHA256, + MediaType: output.MediaType, + } + } + return outputs +} + +func delegationTaskWithInputAliases(task string, inputs []delegateInput, staged []DelegationArtifact) string { + if len(staged) == 0 { + return task + } + aliases := make([]string, 0, len(staged)) + for i, artifact := range staged { + for _, reference := range inputs[i].taskReferences { + task = replaceExactPathReference(task, reference, artifact.Path) + } + source := inputs[i].relativePath + if source == "" || source == artifact.Path { + aliases = append(aliases, artifact.Path) + continue + } + aliases = append(aliases, source+" => "+artifact.Path) + } + return task + "\n\nRead-only delegation inputs: " + strings.Join(aliases, ", ") +} + +func replaceExactPathReference(text, reference, replacement string) string { + if reference == "" || reference == replacement { + return text + } + var rewritten strings.Builder + remaining := text + for { + index := strings.Index(remaining, reference) + if index < 0 { + rewritten.WriteString(remaining) + return rewritten.String() + } + beforeOK := index == 0 || !isPathReferenceByte(remaining[index-1]) + afterIndex := index + len(reference) + afterOK := afterIndex == len(remaining) || !isPathReferenceByte(remaining[afterIndex]) + if beforeOK && afterOK { + rewritten.WriteString(remaining[:index]) + rewritten.WriteString(replacement) + remaining = remaining[afterIndex:] + continue + } + rewritten.WriteString(remaining[:index+len(reference)]) + remaining = remaining[index+len(reference):] + } +} + +func isPathReferenceByte(value byte) bool { + switch { + case value >= 'a' && value <= 'z', + value >= 'A' && value <= 'Z', + value >= '0' && value <= '9': + return true + } + switch value { + case '_', '-', '.', '/', '\\': + return true + default: + return false + } +} + +func publicationMedia(callerWorkspace string, publication DelegationArtifactPublication) []bus.MediaFile { + if len(publication.Manifest.Outputs) == 0 { + return nil + } + media := make([]bus.MediaFile, len(publication.Manifest.Outputs)) + for i, output := range publication.Manifest.Outputs { + durableRelative := filepath.FromSlash(filepath.Join(publication.RootPath, output.Path)) + media[i] = bus.MediaFile{ + Path: filepath.Join(callerWorkspace, durableRelative), + MimeType: output.MediaType, + Filename: filepath.Base(output.Path), + } + } + return media +} + +func redactDelegationArtifactError(err error, exchange *DelegationArtifactExchange) error { + if err == nil { + return nil + } + return &delegationRedactedError{ + message: redactDelegationArtifactText(err.Error(), exchange), + cause: err, + } +} + +type delegationRedactedError struct { + message string + cause error +} + +func (e *delegationRedactedError) Error() string { return e.message } +func (e *delegationRedactedError) Unwrap() error { return e.cause } + +func redactDelegationArtifactText(text string, exchange *DelegationArtifactExchange) string { + replacements := []struct { + hostPath string + alias string + }{ + {exchange.InputsMount().HostRoot, "inputs"}, + {exchange.OutputsHostPath(), "outputs"}, + {exchange.hostRoot, "delegation exchange"}, + } + for _, replacement := range replacements { + for _, variant := range artifactPathRedactionVariants(replacement.hostPath) { + text = strings.ReplaceAll(text, variant, replacement.alias) + } + } + return text +} + +func artifactPathRedactionVariants(hostPath string) []string { + variants := []string{ + hostPath, + filepath.ToSlash(hostPath), + filepath.FromSlash(hostPath), + strings.ReplaceAll(hostPath, `\`, "/"), + strings.ReplaceAll(hostPath, "/", `\`), + } + seen := make(map[string]struct{}, len(variants)) + result := make([]string, 0, len(variants)) + for _, variant := range variants { + if variant == "" { + continue + } + if _, ok := seen[variant]; ok { + continue + } + seen[variant] = struct{}{} + result = append(result, variant) + } + return result +} + +func delegationEventTask( + task string, + callerWorkspace string, + exchange *DelegationArtifactExchange, +) string { + task = redactDelegationArtifactText(task, exchange) + if callerWorkspace != "" { + task = strings.ReplaceAll(task, callerWorkspace, "caller workspace") + task = strings.ReplaceAll( + task, + filepath.ToSlash(callerWorkspace), + "caller workspace", + ) + } + return task +} + +func delegateAdmissionConstraints(ctx context.Context, req DelegateRequest) orchestration.ChildRunConstraints { + parentTaskID, depth := childRunContinuationLineage( + ctx, + subagentTaskIDFromContext(ctx), + subagentDepthFromContext(ctx, 0)+1, + ) + return orchestration.ChildRunConstraints{ + TenantID: parseUUIDOrNil(req.TenantID), + RootAgentID: uuid.Nil, + TaskID: req.DelegationID, + ParentTaskID: parentTaskID, + Depth: depth, + } +} + // announceToParent delivers the delegate result back to the parent agent's // conversation via msgBus, following the same pattern as subagent announce. -func (t *DelegateTool) announceToParent(req DelegateRequest, content string, media []bus.MediaFile) { +func (t *DelegateTool) announceToParent(req DelegateRequest, content string, media []bus.MediaFile) bool { if t.msgBus == nil || req.ChatID == "" { - return + return false } tenantUUID, _ := uuid.Parse(req.TenantID) meta := map[string]string{ @@ -297,6 +1155,8 @@ func (t *DelegateTool) announceToParent(req DelegateRequest, content string, med "delegate_from": req.FromAgentKey, "delegate_to": req.ToAgentKey, MetaParentAgent: req.FromAgentKey, + MetaOriginTraceID: req.OriginTraceID.String(), + MetaOriginRootSpanID: req.OriginRootSpanID.String(), } if req.SenderID != "" { meta[MetaOriginSenderID] = req.SenderID @@ -307,7 +1167,7 @@ func (t *DelegateTool) announceToParent(req DelegateRequest, content string, med if req.UserID != "" { meta[MetaOriginUserID] = req.UserID } - t.msgBus.PublishInbound(bus.InboundMessage{ + return PublishAsyncCompletion(context.Background(), t.msgBus, bus.InboundMessage{ Channel: "system", SenderID: fmt.Sprintf("subagent:delegate:%s", req.DelegationID), ChatID: req.ChatID, @@ -342,4 +1202,3 @@ func (t *DelegateTool) emitEvent(ctx context.Context, eventType eventbus.EventTy Payload: payload, }) } - diff --git a/internal/tools/delegate_tool_hooks_test.go b/internal/tools/delegate_tool_hooks_test.go index 757cd299..72f5128a 100644 --- a/internal/tools/delegate_tool_hooks_test.go +++ b/internal/tools/delegate_tool_hooks_test.go @@ -2,12 +2,26 @@ package tools import ( "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" "testing" + "time" "github.com/google/uuid" + "github.com/nextlevelbuilder/goclaw/internal/bus" + orchestration "github.com/nextlevelbuilder/goclaw/internal/childrun" + "github.com/nextlevelbuilder/goclaw/internal/config" + "github.com/nextlevelbuilder/goclaw/internal/eventbus" "github.com/nextlevelbuilder/goclaw/internal/hooks" "github.com/nextlevelbuilder/goclaw/internal/store" + "github.com/nextlevelbuilder/goclaw/internal/tracing" ) // fakeDispatcher records Fire calls and returns a preset decision. @@ -16,6 +30,34 @@ type fakeDispatcher struct { calls int } +type recordingDelegateEventBus struct { + mu sync.Mutex + events []eventbus.DomainEvent +} + +func (b *recordingDelegateEventBus) Publish(event eventbus.DomainEvent) { + b.mu.Lock() + defer b.mu.Unlock() + b.events = append(b.events, event) +} + +func (*recordingDelegateEventBus) Subscribe( + eventbus.EventType, + eventbus.DomainEventHandler, +) func() { + return func() {} +} + +func (*recordingDelegateEventBus) Start(context.Context) {} + +func (*recordingDelegateEventBus) Drain(time.Duration) error { return nil } + +func (b *recordingDelegateEventBus) snapshot() []eventbus.DomainEvent { + b.mu.Lock() + defer b.mu.Unlock() + return append([]eventbus.DomainEvent(nil), b.events...) +} + func (f *fakeDispatcher) Fire(_ context.Context, _ hooks.Event) (hooks.FireResult, error) { f.calls++ return hooks.FireResult{Decision: f.decision}, nil @@ -25,6 +67,8 @@ func (f *fakeDispatcher) Fire(_ context.Context, _ hooks.Event) (hooks.FireResul type noopAgentLink struct{} +var noopAgentLinkID = uuid.MustParse("8d9c2f78-7914-4e4e-b75f-e6e22ea75921") + func (noopAgentLink) CreateLink(_ context.Context, _ *store.AgentLinkData) error { return nil } func (noopAgentLink) DeleteLink(_ context.Context, _ uuid.UUID) error { return nil } func (noopAgentLink) UpdateLink(_ context.Context, _ uuid.UUID, _ map[string]any) error { @@ -43,7 +87,11 @@ func (noopAgentLink) CanDelegate(_ context.Context, _, _ uuid.UUID) (bool, error return true, nil } func (noopAgentLink) GetLinkBetween(_ context.Context, _, _ uuid.UUID) (*store.AgentLinkData, error) { - return nil, nil + return &store.AgentLinkData{ + BaseModel: store.BaseModel{ID: noopAgentLinkID}, + MaxConcurrent: 3, + Status: store.LinkStatusActive, + }, nil } func (noopAgentLink) DelegateTargets(_ context.Context, _ uuid.UUID) ([]store.AgentLinkData, error) { return nil, nil @@ -56,6 +104,16 @@ func (noopAgentLink) SearchDelegateTargetsByEmbedding(_ context.Context, _ uuid. } func (noopAgentLink) DeleteTeamLinksForAgent(_ context.Context, _, _ uuid.UUID) error { return nil } +type fixedAgentLink struct { + noopAgentLink + link store.AgentLinkData +} + +func (f fixedAgentLink) GetLinkBetween(_ context.Context, _, _ uuid.UUID) (*store.AgentLinkData, error) { + link := f.link + return &link, nil +} + type noopAgentCRUD struct { keyToID map[string]uuid.UUID } @@ -81,22 +139,215 @@ func (n noopAgentCRUD) GetByIDs(_ context.Context, _ []uuid.UUID) ([]store.Agent return nil, nil } func (n noopAgentCRUD) Update(_ context.Context, _ uuid.UUID, _ map[string]any) error { return nil } -func (n noopAgentCRUD) Delete(_ context.Context, _ uuid.UUID) error { return nil } -func (n noopAgentCRUD) List(_ context.Context, _ string) ([]store.AgentData, error) { return nil, nil } -func (n noopAgentCRUD) GetDefault(_ context.Context) (*store.AgentData, error) { return nil, nil } -func (n noopAgentCRUD) ResetStuckSummoning(_ context.Context) (int64, error) { return 0, nil } +func (n noopAgentCRUD) Delete(_ context.Context, _ uuid.UUID) error { return nil } +func (n noopAgentCRUD) List(_ context.Context, _ string) ([]store.AgentData, error) { return nil, nil } +func (n noopAgentCRUD) GetDefault(_ context.Context) (*store.AgentData, error) { return nil, nil } +func (n noopAgentCRUD) ResetStuckSummoning(_ context.Context) (int64, error) { return 0, nil } // --- helpers --- -func makeDelegateCtx() context.Context { +func makeDelegateCtx(t *testing.T) context.Context { + t.Helper() ctx := store.WithAgentID(context.Background(), uuid.New()) - ctx = store.WithTenantID(ctx, uuid.New()) + ctx = store.WithTenantID(ctx, store.MasterTenantID) ctx = store.WithAgentKey(ctx, "parent-agent") + ctx = WithToolWorkspace(ctx, t.TempDir()) return ctx } +func newDelegateTestTool( + t *testing.T, + links store.AgentLinkStore, + runFn DelegateRunFunc, +) *DelegateTool { + t.Helper() + tool := NewDelegateTool(links, noopAgentCRUD{}, nil, runFn) + tool.SetWorkspace(t.TempDir()) + t.Cleanup(tool.Close) + return tool +} + // --- tests --- +func TestDelegateTool_EmitsSentOnlyAfterAdmissionWithLogicalTask(t *testing.T) { + t.Run("rejected admission emits nothing", func(t *testing.T) { + admission := orchestration.NewChildRunAdmission(1, 1) + if err := admission.Close(context.Background()); err != nil { + t.Fatal(err) + } + events := &recordingDelegateEventBus{} + tool := NewDelegateToolWithAdmission( + noopAgentLink{}, + noopAgentCRUD{}, + events, + func(_ context.Context, _ DelegateRequest) (DelegateResult, error) { + t.Fatal("runFn called after admission closed") + return DelegateResult{}, nil + }, + admission, + ) + tool.SetWorkspace(t.TempDir()) + defer tool.Close() + + result := tool.Execute(makeDelegateCtx(t), map[string]any{ + "agent_key": "child-agent", + "task": "do something", + "mode": "sync", + }) + if result == nil || !result.IsError { + t.Fatalf("result = %#v, want admission error", result) + } + if got := events.snapshot(); len(got) != 0 { + t.Fatalf("events = %#v, want none", got) + } + }) + + t.Run("accepted dispatch redacts caller host path", func(t *testing.T) { + events := &recordingDelegateEventBus{} + managedWorkspace := t.TempDir() + callerWorkspace := t.TempDir() + tool := NewDelegateTool( + noopAgentLink{}, + noopAgentCRUD{}, + events, + func(_ context.Context, _ DelegateRequest) (DelegateResult, error) { + return DelegateResult{Content: "done"}, nil + }, + ) + tool.SetWorkspace(managedWorkspace) + defer tool.Close() + + ctx := WithToolWorkspace(makeDelegateCtx(t), callerWorkspace) + result := tool.Execute(ctx, map[string]any{ + "agent_key": "child-agent", + "task": "inspect " + filepath.Join(callerWorkspace, "report.txt"), + "mode": "sync", + }) + if result == nil || result.IsError { + t.Fatalf("result = %#v, want success", result) + } + eventsSnapshot := events.snapshot() + if len(eventsSnapshot) == 0 { + t.Fatal("delegate.sent not emitted") + } + var sent *eventbus.DelegateSentPayload + for _, event := range eventsSnapshot { + if event.Type != eventbus.EventDelegateSent { + continue + } + payload, ok := event.Payload.(eventbus.DelegateSentPayload) + if !ok { + t.Fatalf("delegate.sent payload = %T", event.Payload) + } + sent = &payload + break + } + if sent == nil { + t.Fatalf("events = %#v, want delegate.sent", eventsSnapshot) + } + if strings.Contains(sent.Task, callerWorkspace) || + !strings.Contains(sent.Task, "caller workspace") { + t.Fatalf("delegate.sent task = %q", sent.Task) + } + }) +} + +func TestDelegateTool_SyncQueuedTimeoutDoesNotReportCompletion(t *testing.T) { + admission := orchestration.NewChildRunAdmission(1, 4) + blockerStarted := make(chan struct{}) + blockerRelease := make(chan struct{}) + blocker, err := admission.Enqueue( + context.Background(), + orchestration.ChildRunConstraints{TaskID: "blocker"}, + func(context.Context, *orchestration.ChildRunLease) { + close(blockerStarted) + <-blockerRelease + }, + ) + if err != nil { + t.Fatal(err) + } + if err := blocker.Activate(); err != nil { + t.Fatal(err) + } + select { + case <-blockerStarted: + case <-time.After(time.Second): + t.Fatal("blocker did not start") + } + + events := &recordingDelegateEventBus{} + runCalled := false + tool := NewDelegateToolWithAdmission( + noopAgentLink{}, + noopAgentCRUD{}, + events, + func(_ context.Context, _ DelegateRequest) (DelegateResult, error) { + runCalled = true + return DelegateResult{Content: "unexpected"}, nil + }, + admission, + ) + tool.SetWorkspace(t.TempDir()) + defer tool.Close() + ctx, cancel := context.WithTimeout(makeDelegateCtx(t), 20*time.Millisecond) + defer cancel() + result := tool.Execute(ctx, map[string]any{ + "agent_key": "child-agent", + "task": "must time out while queued", + "mode": "sync", + }) + if result == nil || !result.IsError { + t.Fatalf("result = %#v, want queued timeout", result) + } + if runCalled { + t.Fatal("queued timed-out delegation ran") + } + for _, event := range events.snapshot() { + if event.Type == eventbus.EventDelegateSent || + event.Type == eventbus.EventDelegateCompleted { + t.Fatalf("timed-out queued delegation emitted %q", event.Type) + } + } + + close(blockerRelease) + <-blocker.Done() + if err := admission.Close(context.Background()); err != nil { + t.Fatal(err) + } +} + +func TestDelegateTool_SyncCallbackPanicReportsFailure(t *testing.T) { + events := &recordingDelegateEventBus{} + tool := NewDelegateTool( + noopAgentLink{}, + noopAgentCRUD{}, + events, + func(_ context.Context, _ DelegateRequest) (DelegateResult, error) { + panic("injected delegate panic") + }, + ) + tool.SetWorkspace(t.TempDir()) + defer tool.Close() + result := tool.Execute(makeDelegateCtx(t), map[string]any{ + "agent_key": "child-agent", + "task": "panic safely", + "mode": "sync", + }) + if result == nil || !result.IsError || + !strings.Contains(result.ForLLM, "child run panic") { + t.Fatalf("result = %#v, want recovered callback panic", result) + } + var failed, completed bool + for _, event := range events.snapshot() { + failed = failed || event.Type == eventbus.EventDelegateFailed + completed = completed || event.Type == eventbus.EventDelegateCompleted + } + if !failed || completed { + t.Fatalf("events = %#v, want failed without completed", events.snapshot()) + } +} + func TestDelegateTool_SubagentStartBlock_AbortsDispatch(t *testing.T) { runCalled := 0 runFn := func(_ context.Context, _ DelegateRequest) (DelegateResult, error) { @@ -104,11 +355,11 @@ func TestDelegateTool_SubagentStartBlock_AbortsDispatch(t *testing.T) { return DelegateResult{Content: "ok"}, nil } - tool := NewDelegateTool(noopAgentLink{}, noopAgentCRUD{}, nil, runFn) + tool := newDelegateTestTool(t, noopAgentLink{}, runFn) disp := &fakeDispatcher{decision: hooks.DecisionBlock} tool.SetHookDispatcher(disp) - result := tool.Execute(makeDelegateCtx(), map[string]any{ + result := tool.Execute(makeDelegateCtx(t), map[string]any{ "agent_key": "child-agent", "task": "do something", "mode": "sync", @@ -132,11 +383,11 @@ func TestDelegateTool_SubagentStartAllow_ProceedsToRun(t *testing.T) { return DelegateResult{Content: "done"}, nil } - tool := NewDelegateTool(noopAgentLink{}, noopAgentCRUD{}, nil, runFn) + tool := newDelegateTestTool(t, noopAgentLink{}, runFn) disp := &fakeDispatcher{decision: hooks.DecisionAllow} tool.SetHookDispatcher(disp) - result := tool.Execute(makeDelegateCtx(), map[string]any{ + result := tool.Execute(makeDelegateCtx(t), map[string]any{ "agent_key": "child-agent", "task": "do something", "mode": "sync", @@ -158,9 +409,9 @@ func TestDelegateTool_NilDispatcher_SkipsHook(t *testing.T) { } // No SetHookDispatcher — hookDispatcher stays nil. - tool := NewDelegateTool(noopAgentLink{}, noopAgentCRUD{}, nil, runFn) + tool := newDelegateTestTool(t, noopAgentLink{}, runFn) - result := tool.Execute(makeDelegateCtx(), map[string]any{ + result := tool.Execute(makeDelegateCtx(t), map[string]any{ "agent_key": "child-agent", "task": "do something", "mode": "sync", @@ -173,3 +424,1753 @@ func TestDelegateTool_NilDispatcher_SkipsHook(t *testing.T) { t.Errorf("expected runFn called once; got %d", runCalled) } } + +func TestDelegateTool_SyncNestedRunCompletesBeforeOuterPublication(t *testing.T) { + admission := orchestration.NewChildRunAdmission(1, 8) + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := admission.Close(ctx); err != nil { + t.Errorf("close admission: %v", err) + } + }) + + callerWorkspace := t.TempDir() + managedWorkspace := t.TempDir() + childB := uuid.New() + childCStarted := make(chan struct{}) + releaseChildC := make(chan struct{}) + var childCCompleted atomic.Bool + + var tool *DelegateTool + runFn := func(ctx context.Context, req DelegateRequest) (DelegateResult, error) { + switch req.ToAgentKey { + case "child-b": + nestedCtx := store.WithAgentID(ctx, childB) + nestedCtx = store.WithAgentKey(nestedCtx, "child-b") + nestedCtx = WithToolAgentKey(nestedCtx, "child-b") + nestedCtx = WithToolWorkspace(nestedCtx, req.DelegateOutputsPath) + nestedCtx = WithDelegationID(nestedCtx, req.DelegationID) + nestedCtx = WithDelegationArtifactInputs(nestedCtx, req.DelegateInputsPath) + nestedCtx = WithSubagentConfig(nestedCtx, &config.SubagentsConfig{MaxSpawnDepth: 1}) + if depth := subagentDepthFromContext(nestedCtx, -1); depth != 0 { + return DelegateResult{}, fmt.Errorf("delegated agent depth = %d, want 0", depth) + } + if parentTaskID := subagentTaskIDFromContext(nestedCtx); parentTaskID != "" { + return DelegateResult{}, fmt.Errorf("delegated agent parent task = %q, want empty", parentTaskID) + } + if scope := subagentScopeFromContext(nestedCtx); scope.RootAgentID != childB || + scope.RootAgentKey != "child-b" { + return DelegateResult{}, fmt.Errorf("delegated agent scope = %#v, want child-b", scope) + } + + nested := tool.Execute(nestedCtx, map[string]any{ + "agent_key": "child-c", + "task": "finish before child B returns", + "mode": "sync", + }) + if nested == nil || nested.IsError { + return DelegateResult{}, fmt.Errorf("nested delegation failed: %#v", nested) + } + if !childCCompleted.Load() { + return DelegateResult{}, errors.New("nested delegation returned before child C completed") + } + if err := os.WriteFile( + filepath.Join(req.DelegateOutputsPath, "b-after-c.txt"), + []byte("child C completed"), + 0600, + ); err != nil { + return DelegateResult{}, err + } + return DelegateResult{Content: "child B completed after child C"}, nil + + case "child-c": + close(childCStarted) + select { + case <-releaseChildC: + case <-ctx.Done(): + return DelegateResult{}, ctx.Err() + } + childCCompleted.Store(true) + return DelegateResult{Content: "child C completed"}, nil + + default: + return DelegateResult{}, fmt.Errorf("unexpected target %q", req.ToAgentKey) + } + } + + tool = NewDelegateToolWithAdmission( + noopAgentLink{}, + noopAgentCRUD{keyToID: map[string]uuid.UUID{ + "child-b": childB, + "child-c": uuid.New(), + }}, + nil, + runFn, + admission, + ) + tool.SetWorkspace(managedWorkspace) + t.Cleanup(tool.Close) + + ctx := WithToolWorkspace(makeDelegateCtx(t), callerWorkspace) + ctx = WithSubagentConfig(ctx, &config.SubagentsConfig{MaxSpawnDepth: 1}) + resultCh := make(chan *Result, 1) + go func() { + resultCh <- tool.Execute(ctx, map[string]any{ + "agent_key": "child-b", + "task": "delegate synchronously to child C", + "mode": "sync", + }) + }() + + select { + case <-childCStarted: + case <-time.After(time.Second): + t.Fatal("nested child C did not start") + } + select { + case result := <-resultCh: + t.Fatalf("outer delegation published before nested child completed: %#v", result) + case <-time.After(50 * time.Millisecond): + } + if entries, err := os.ReadDir(filepath.Join(callerWorkspace, ".delegations")); err == nil && len(entries) > 0 { + t.Fatalf("outer artifacts visible while nested child was active: %#v", entries) + } else if err != nil && !errors.Is(err, os.ErrNotExist) { + t.Fatalf("inspect caller publication root: %v", err) + } + + close(releaseChildC) + var result *Result + select { + case result = <-resultCh: + case <-time.After(2 * time.Second): + t.Fatal("outer delegation did not finish after nested child completed") + } + if result == nil || result.IsError { + t.Fatalf("outer delegation = %#v, want success", result) + } + foundMarker := false + for _, media := range result.Media { + if media.Filename == "b-after-c.txt" { + foundMarker = true + content, err := os.ReadFile(media.Path) + if err != nil { + t.Fatalf("read published marker: %v", err) + } + if string(content) != "child C completed" { + t.Fatalf("published marker = %q", content) + } + } + } + if !foundMarker { + t.Fatalf("published media = %#v, want b-after-c.txt", result.Media) + } +} + +func TestDelegateTool_ConcurrentMultiLinkTopologiesPublishInCallerScope(t *testing.T) { + managedWorkspace := t.TempDir() + callerA := t.TempDir() + callerC := t.TempDir() + agentA := uuid.New() + agentB := uuid.New() + agentC := uuid.New() + + started := make(chan struct{}, 3) + release := make(chan struct{}) + runFn := func(ctx context.Context, req DelegateRequest) (DelegateResult, error) { + started <- struct{}{} + select { + case <-release: + case <-ctx.Done(): + return DelegateResult{}, ctx.Err() + } + if err := os.WriteFile( + filepath.Join(req.DelegateOutputsPath, "result.txt"), + []byte(req.Task), + 0600, + ); err != nil { + return DelegateResult{}, err + } + return DelegateResult{Content: req.Task}, nil + } + tool := NewDelegateTool( + noopAgentLink{}, + noopAgentCRUD{keyToID: map[string]uuid.UUID{ + "agent-b": agentB, + "agent-c": agentC, + }}, + nil, + runFn, + ) + tool.SetWorkspace(managedWorkspace) + t.Cleanup(tool.Close) + + type topologyCase struct { + name string + fromID uuid.UUID + fromKey string + toKey string + callerWorkspace string + payload string + } + cases := []topologyCase{ + { + name: "a-to-b", + fromID: agentA, + fromKey: "agent-a", + toKey: "agent-b", + callerWorkspace: callerA, + payload: "A to B", + }, + { + name: "a-to-c", + fromID: agentA, + fromKey: "agent-a", + toKey: "agent-c", + callerWorkspace: callerA, + payload: "A to C", + }, + { + name: "c-to-b", + fromID: agentC, + fromKey: "agent-c", + toKey: "agent-b", + callerWorkspace: callerC, + payload: "C to B", + }, + } + + results := make([]*Result, len(cases)) + var wg sync.WaitGroup + for i := range cases { + wg.Add(1) + go func(i int) { + defer wg.Done() + tc := cases[i] + ctx := store.WithAgentID(context.Background(), tc.fromID) + ctx = store.WithAgentKey(ctx, tc.fromKey) + ctx = store.WithTenantID(ctx, store.MasterTenantID) + ctx = WithToolWorkspace(ctx, tc.callerWorkspace) + results[i] = tool.Execute(ctx, map[string]any{ + "agent_key": tc.toKey, + "task": tc.payload, + "mode": "sync", + }) + }(i) + } + for range cases { + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("concurrent delegation did not start") + } + } + close(release) + wg.Wait() + + aPublications := make(map[string]struct{}) + for i, tc := range cases { + result := results[i] + if result == nil || result.IsError { + t.Fatalf("%s result = %#v", tc.name, result) + } + if len(result.Media) != 1 { + t.Fatalf("%s media = %#v, want one output", tc.name, result.Media) + } + media := result.Media[0] + relative, err := filepath.Rel(tc.callerWorkspace, media.Path) + if err != nil || strings.HasPrefix(relative, "..") { + t.Fatalf("%s output %q escaped caller %q", tc.name, media.Path, tc.callerWorkspace) + } + parts := strings.Split(filepath.ToSlash(relative), "/") + if len(parts) != 4 || parts[0] != ".delegations" || parts[2] != "outputs" || parts[3] != "result.txt" { + t.Fatalf("%s relative output = %q", tc.name, relative) + } + content, err := os.ReadFile(media.Path) + if err != nil { + t.Fatalf("%s read output: %v", tc.name, err) + } + if string(content) != tc.payload { + t.Fatalf("%s output = %q, want %q", tc.name, content, tc.payload) + } + if tc.fromKey == "agent-a" { + if _, exists := aPublications[parts[1]]; exists { + t.Fatalf("A→B and A→C reused delegation %q", parts[1]) + } + aPublications[parts[1]] = struct{}{} + } + } + if len(aPublications) != 2 { + t.Fatalf("A publications = %d, want 2 isolated delegation roots", len(aPublications)) + } +} + +func TestDelegateTool_LinkMaxConcurrentIsReservedAndUnenforced(t *testing.T) { + linkID := uuid.New() + started := make(chan struct{}) + releaseRun := make(chan struct{}) + var calls atomic.Int32 + runFn := func(_ context.Context, _ DelegateRequest) (DelegateResult, error) { + call := calls.Add(1) + if call == 1 { + close(started) + } + if call <= 2 { + <-releaseRun + } + return DelegateResult{Content: "done"}, nil + } + tool := newDelegateTestTool(t, fixedAgentLink{link: store.AgentLinkData{ + BaseModel: store.BaseModel{ID: linkID}, + MaxConcurrent: 1, + Status: store.LinkStatusActive, + }}, runFn) + + first := tool.Execute(makeDelegateCtx(t), map[string]any{ + "agent_key": "child-agent", + "task": "first", + "mode": "async", + }) + if first == nil || first.IsError { + t.Fatalf("first delegation = %#v, want accepted", first) + } + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("first async delegation did not start") + } + + second := tool.Execute(makeDelegateCtx(t), map[string]any{ + "agent_key": "child-agent", + "task": "second", + "mode": "async", + }) + if second == nil || second.IsError { + t.Fatalf("second delegation = %#v, want shared admission to accept it", second) + } + deadline := time.Now().Add(time.Second) + for calls.Load() < 2 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if got := calls.Load(); got != 2 { + t.Fatalf("runFn calls = %d, want max_concurrent ignored", got) + } + + close(releaseRun) + + third := tool.Execute(makeDelegateCtx(t), map[string]any{ + "agent_key": "child-agent", + "task": "third", + "mode": "sync", + }) + if third == nil || third.IsError { + t.Fatalf("third delegation = %#v, want success after release", third) + } +} + +func TestDelegateTool_LinkSlotReleasedOnHookBlockAndSyncError(t *testing.T) { + link := fixedAgentLink{link: store.AgentLinkData{ + BaseModel: store.BaseModel{ID: uuid.New()}, + MaxConcurrent: 1, + Status: store.LinkStatusActive, + }} + var calls atomic.Int32 + runFn := func(_ context.Context, _ DelegateRequest) (DelegateResult, error) { + if calls.Add(1) == 1 { + return DelegateResult{}, errors.New("first run failed") + } + return DelegateResult{Content: "done"}, nil + } + tool := newDelegateTestTool(t, link, runFn) + disp := &fakeDispatcher{decision: hooks.DecisionBlock} + tool.SetHookDispatcher(disp) + + blocked := tool.Execute(makeDelegateCtx(t), map[string]any{ + "agent_key": "child-agent", + "task": "blocked", + "mode": "sync", + }) + if blocked == nil || !blocked.IsError { + t.Fatalf("blocked delegation = %#v, want error", blocked) + } + + disp.decision = hooks.DecisionAllow + failed := tool.Execute(makeDelegateCtx(t), map[string]any{ + "agent_key": "child-agent", + "task": "fails", + "mode": "sync", + }) + if failed == nil || !failed.IsError || + strings.Contains(failed.ForLLM, "active delegation limit reached") { + t.Fatalf("failed delegation = %#v, want run error after hook slot release", failed) + } + + succeeded := tool.Execute(makeDelegateCtx(t), map[string]any{ + "agent_key": "child-agent", + "task": "succeeds", + "mode": "sync", + }) + if succeeded == nil || succeeded.IsError { + t.Fatalf("delegation after sync error = %#v, want success", succeeded) + } +} + +func TestDelegateTool_PreservesGroupAuthorizationScope(t *testing.T) { + var captured DelegateRequest + runFn := func(_ context.Context, req DelegateRequest) (DelegateResult, error) { + captured = req + return DelegateResult{Content: "done"}, nil + } + tool := newDelegateTestTool(t, noopAgentLink{}, runFn) + + ctx := makeDelegateCtx(t) + ctx = store.WithUserID(ctx, "group:telegram:-100123") + ctx = store.WithSenderID(ctx, "386246614") + ctx = store.WithRole(ctx, "viewer") + ctx = WithToolChannel(ctx, "telegram-main") + ctx = WithToolChannelType(ctx, "telegram") + ctx = WithToolChatID(ctx, "-100123") + ctx = WithToolPeerKind(ctx, "group") + + result := tool.Execute(ctx, map[string]any{ + "agent_key": "child-agent", + "task": "create a file", + "mode": "sync", + }) + + if result == nil || result.IsError { + t.Fatalf("delegate result = %#v, want success", result) + } + if captured.UserID != "group:telegram:-100123" { + t.Fatalf("UserID = %q, want group authorization scope", captured.UserID) + } + if captured.SenderID != "386246614" || captured.Role != "viewer" { + t.Fatalf("actor scope = (%q, %q), want real sender and role", captured.SenderID, captured.Role) + } + if captured.Channel != "telegram-main" || captured.ChannelType != "telegram" || + captured.ChatID != "-100123" || captured.PeerKind != "group" { + t.Fatalf("origin scope = %#v", captured) + } +} + +func TestDelegateTool_CurrentRunMediaBecomesReadOnlyRelativeInput(t *testing.T) { + workspace := t.TempDir() + source := filepath.Join(workspace, "reference.jpg") + if err := os.WriteFile(source, []byte("reference"), 0644); err != nil { + t.Fatal(err) + } + + var captured DelegateRequest + runFn := func(_ context.Context, req DelegateRequest) (DelegateResult, error) { + captured = req + staged, err := os.ReadFile(filepath.Join(req.DelegateInputsPath, "reference.jpg")) + if err != nil { + return DelegateResult{}, err + } + if string(staged) != "reference" { + return DelegateResult{}, errors.New("staged input content mismatch") + } + return DelegateResult{Content: "done"}, nil + } + tool := newDelegateTestTool(t, noopAgentLink{}, runFn) + + ctx := WithToolWorkspace(makeDelegateCtx(t), workspace) + ctx = WithRunMediaPaths(ctx, []string{source}) + result := tool.Execute(ctx, map[string]any{ + "agent_key": "child-agent", + "task": "use " + source, + "mode": "sync", + }) + + if result == nil || result.IsError { + t.Fatalf("delegate result = %#v, want success", result) + } + if strings.Contains(captured.Task, source) || !strings.Contains(captured.Task, "inputs/reference.jpg") { + t.Fatalf("Task = %q, want only logical input alias", captured.Task) + } + if captured.DelegateInputsPath == "" || captured.DelegateOutputsPath == "" { + t.Fatalf("runtime artifact roots missing: %#v", captured) + } +} + +func TestDelegateTool_DoesNotScrapeAbsolutePathsFromTask(t *testing.T) { + workspace := t.TempDir() + source := filepath.Join(workspace, "family-reference.jpg") + if err := os.WriteFile(source, []byte("reference"), 0644); err != nil { + t.Fatal(err) + } + + var captured DelegateRequest + var stagedInputCount int + runFn := func(_ context.Context, req DelegateRequest) (DelegateResult, error) { + captured = req + entries, err := os.ReadDir(req.DelegateInputsPath) + if err != nil { + return DelegateResult{}, err + } + stagedInputCount = len(entries) + return DelegateResult{Content: "done"}, nil + } + tool := newDelegateTestTool(t, noopAgentLink{}, runFn) + + ctx := WithToolWorkspace(makeDelegateCtx(t), workspace) + result := tool.Execute(ctx, map[string]any{ + "agent_key": "child-agent", + "task": "Retry using the exact sample file path: " + source + ".", + "mode": "sync", + }) + + if result == nil || result.IsError { + t.Fatalf("delegate result = %#v, want success", result) + } + if stagedInputCount != 0 { + t.Fatalf("staged input count = %d, want no free-form path scraping", stagedInputCount) + } + if !strings.Contains(captured.Task, source) { + t.Fatalf("Task = %q, want untrusted prose left as prose", captured.Task) + } +} + +func TestDelegateTool_RejectsInvalidExplicitInputBeforeDispatch(t *testing.T) { + workspace := t.TempDir() + + var called atomic.Bool + runFn := func(_ context.Context, _ DelegateRequest) (DelegateResult, error) { + called.Store(true) + return DelegateResult{Content: "done"}, nil + } + tool := newDelegateTestTool(t, noopAgentLink{}, runFn) + + ctx := WithToolWorkspace(makeDelegateCtx(t), workspace) + result := tool.Execute(ctx, map[string]any{ + "agent_key": "child-agent", + "task": "Use the requested input", + "mode": "sync", + "inputs": []any{"../outside-reference.jpg"}, + }) + + if result == nil || !result.IsError { + t.Fatalf("delegate result = %#v, want validation error", result) + } + if called.Load() { + t.Fatal("runFn called for invalid explicit input") + } +} + +func TestDelegateTool_MissingManagedWorkspaceFailsClosed(t *testing.T) { + tool := NewDelegateTool(noopAgentLink{}, noopAgentCRUD{}, nil, func(_ context.Context, _ DelegateRequest) (DelegateResult, error) { + return DelegateResult{Content: "unexpected"}, nil + }) + result := tool.Execute(makeDelegateCtx(t), map[string]any{ + "agent_key": "child-agent", + "task": "do something", + "mode": "sync", + }) + if result == nil || !result.IsError { + t.Fatalf("delegate result = %#v, want fail-closed setup error", result) + } +} + +func TestDelegateTool_PublishesOnlyManifestOutputsAndCleansExchange(t *testing.T) { + workspace := t.TempDir() + managedWorkspace := t.TempDir() + rawMedia := filepath.Join(t.TempDir(), "raw-child-media.png") + if err := os.WriteFile(rawMedia, []byte("raw"), 0644); err != nil { + t.Fatal(err) + } + + var captured DelegateRequest + runFn := func(_ context.Context, req DelegateRequest) (DelegateResult, error) { + captured = req + if err := os.WriteFile(filepath.Join(req.DelegateOutputsPath, "report.txt"), []byte("published"), 0600); err != nil { + return DelegateResult{}, err + } + return DelegateResult{ + Content: "done", + Media: []bus.MediaFile{{ + Path: rawMedia, + MimeType: "image/png", + Filename: "raw-child-media.png", + }}, + }, nil + } + tool := NewDelegateTool(noopAgentLink{}, noopAgentCRUD{}, nil, runFn) + tool.SetWorkspace(managedWorkspace) + + ctx := WithToolWorkspace(makeDelegateCtx(t), workspace) + result := tool.Execute(ctx, map[string]any{ + "agent_key": "child-agent", + "task": "create report.txt", + "mode": "sync", + }) + if result == nil || result.IsError { + t.Fatalf("delegate result = %#v, want success", result) + } + if len(result.Media) != 1 { + t.Fatalf("result media = %#v, want one published manifest output", result.Media) + } + got := result.Media[0] + if got.Path == rawMedia || got.Filename != "report.txt" || got.MimeType != "text/plain; charset=utf-8" { + t.Fatalf("published media = %#v", got) + } + if content, err := os.ReadFile(got.Path); err != nil || string(content) != "published" { + t.Fatalf("published content = %q, %v", content, err) + } + if _, err := os.Stat(filepath.Dir(captured.DelegateOutputsPath)); !os.IsNotExist(err) { + t.Fatalf("successful exchange retained at runtime path: %v", err) + } +} + +func TestDelegateTool_AdmissionConstraintsAreTenantScopedAndRootless(t *testing.T) { + tenantID := uuid.New() + ctx := store.WithTenantID(context.Background(), tenantID) + ctx = withSubagentExecution(ctx, TaskScope{ + TenantID: tenantID, RootAgentID: uuid.New(), RootAgentKey: "parent-agent", + }, "parent-task", 3, nil) + req := DelegateRequest{ + TenantID: tenantID.String(), + DelegationID: uuid.NewString(), + } + + got := delegateAdmissionConstraints(ctx, req) + if got.TenantID != tenantID || got.RootAgentID != uuid.Nil { + t.Fatalf("identity constraints = %#v", got) + } + if got.TaskID != req.DelegationID || got.ParentTaskID != "parent-task" { + t.Fatalf("task constraints = %#v", got) + } + if got.Depth != 4 || got.MaxDepth != 0 { + t.Fatalf("depth constraints = %#v", got) + } +} + +func TestDelegateTool_AdmissionDoesNotUseAgentSpawnDepth(t *testing.T) { + req := DelegateRequest{ + TenantID: store.MasterTenantID.String(), + DelegationID: uuid.NewString(), + } + got := delegateAdmissionConstraints(context.Background(), req) + if got.Depth != 1 || got.MaxDepth != 0 { + t.Fatalf("top-level admission depth = %#v, want structural depth 1 without agent cap", got) + } + + ctx := WithSubagentConfig(context.Background(), &config.SubagentsConfig{MaxSpawnDepth: 4}) + got = delegateAdmissionConstraints(ctx, req) + if got.MaxDepth != 0 { + t.Fatalf("configured agent spawn depth leaked into admission = %#v", got) + } +} + +func TestDelegateTool_StagesArtifactsOnlyAfterAdmission(t *testing.T) { + workspace := t.TempDir() + managedWorkspace := t.TempDir() + if err := os.WriteFile(filepath.Join(workspace, "input.txt"), []byte("input"), 0600); err != nil { + t.Fatal(err) + } + + admission := orchestration.NewChildRunAdmission(1, 4) + defer func() { + if err := admission.Close(context.Background()); err != nil { + t.Errorf("close admission: %v", err) + } + }() + firstStarted := make(chan struct{}) + releaseFirst := make(chan struct{}) + secondStarted := make(chan struct{}) + runFn := func(_ context.Context, req DelegateRequest) (DelegateResult, error) { + if strings.HasPrefix(req.Task, "first") { + close(firstStarted) + <-releaseFirst + } else { + close(secondStarted) + } + return DelegateResult{Content: "done"}, nil + } + tool := NewDelegateToolWithAdmission(noopAgentLink{}, noopAgentCRUD{}, nil, runFn, admission) + tool.SetWorkspace(managedWorkspace) + ctx := WithToolWorkspace(makeDelegateCtx(t), workspace) + + first := tool.Execute(ctx, map[string]any{ + "agent_key": "child-agent", + "task": "first", + "mode": "async", + }) + if first == nil || first.IsError { + t.Fatalf("first delegation = %#v", first) + } + select { + case <-firstStarted: + case <-time.After(time.Second): + t.Fatal("first delegation did not start") + } + + second := tool.Execute(ctx, map[string]any{ + "agent_key": "child-agent", + "task": "second", + "mode": "async", + "inputs": []any{"input.txt"}, + }) + if second == nil || second.IsError { + t.Fatalf("second delegation = %#v", second) + } + var accepted struct { + DelegationID string `json:"delegation_id"` + } + if err := json.Unmarshal([]byte(second.ForLLM), &accepted); err != nil { + t.Fatal(err) + } + secondExchange := filepath.Join( + managedWorkspace, + "collaboration", + "delegations", + accepted.DelegationID, + ) + if _, err := os.Stat(secondExchange); !os.IsNotExist(err) { + t.Fatalf("pending delegation staged before admission: %v", err) + } + + close(releaseFirst) + select { + case <-secondStarted: + case <-time.After(time.Second): + t.Fatal("second delegation did not start after admission") + } +} + +func TestDelegateTool_AsyncReleasesAdmissionBeforeAnnouncement(t *testing.T) { + admission := orchestration.NewChildRunAdmission(1, 4) + messageBus := bus.New() + for range 1000 { + messageBus.PublishInbound(bus.InboundMessage{Content: "fill"}) + } + + firstStarted := make(chan struct{}) + secondStarted := make(chan struct{}) + var calls atomic.Int32 + runFn := func(_ context.Context, _ DelegateRequest) (DelegateResult, error) { + switch calls.Add(1) { + case 1: + close(firstStarted) + case 2: + close(secondStarted) + } + return DelegateResult{Content: "done"}, nil + } + tool := NewDelegateToolWithAdmission(noopAgentLink{}, noopAgentCRUD{}, nil, runFn, admission) + tool.SetWorkspace(t.TempDir()) + tool.SetMsgBus(messageBus) + taskStore := newRecordingSubagentTaskStore() + tool.SetTaskStore(taskStore) + if tool.announceToParent(DelegateRequest{ + DelegationID: "backpressure-probe", + ChatID: "chat-1", + }, "must not block", nil) { + t.Fatal("full message bus accepted an announcement") + } + + ctx := WithToolChatID(makeDelegateCtx(t), "chat-1") + ctx = WithToolSessionKey(ctx, "delegate-parent-session") + first := tool.Execute(ctx, map[string]any{ + "agent_key": "child-agent", + "task": "first", + "mode": "async", + }) + if first == nil || first.IsError { + t.Fatalf("first delegation = %#v, want accepted", first) + } + select { + case <-firstStarted: + case <-time.After(time.Second): + t.Fatal("first delegation did not start") + } + + second := tool.Execute(ctx, map[string]any{ + "agent_key": "child-agent", + "task": "second", + "mode": "async", + }) + if second == nil || second.IsError { + t.Fatalf("second delegation = %#v, want accepted", second) + } + select { + case <-secondStarted: + case <-time.After(time.Second): + t.Fatal("blocked announce retained the only child-run permit") + } + for range 2 { + select { + case metadata := <-taskStore.metadata: + if metadata[asyncCompletionDeliveryKey] != asyncCompletionDeliveryMissed { + t.Fatalf("announcement metadata = %#v, want undelivered", metadata) + } + case <-time.After(time.Second): + t.Fatal("delegation missed announcement was not recorded after bus saturation") + } + } + + if err := admission.Close(context.Background()); err != nil { + t.Fatalf("close admission: %v", err) + } + tool.Close() +} + +func TestDelegateToolCloseContextDrainsAsyncCompletion(t *testing.T) { + started := make(chan struct{}) + release := make(chan struct{}) + tool := newDelegateTestTool(t, noopAgentLink{}, func(_ context.Context, _ DelegateRequest) (DelegateResult, error) { + close(started) + <-release + return DelegateResult{Content: "done"}, nil + }) + tool.SetTaskStore(newRecordingSubagentTaskStore()) + + result := tool.Execute(makeDelegateCtx(t), map[string]any{ + "agent_key": "child-agent", + "task": "wait for shutdown drain", + "mode": "async", + }) + if result == nil || result.IsError { + t.Fatalf("delegate result = %#v, want accepted", result) + } + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("delegation did not start") + } + + timeoutCtx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + if err := tool.CloseContext(timeoutCtx); err == nil { + t.Fatal("CloseContext returned before async completion finished") + } + + close(release) + if err := tool.CloseContext(context.Background()); err != nil { + t.Fatalf("CloseContext after completion: %v", err) + } +} + +func TestDelegationTaskWithInputAliasesDoesNotRewriteProseSubstrings(t *testing.T) { + task := "Use the data file to update the database metadata." + inputs := []delegateInput{{ + relativePath: "data", + }} + staged := []DelegationArtifact{{Path: "inputs/data"}} + + got := delegationTaskWithInputAliases(task, inputs, staged) + if !strings.HasPrefix(got, task) { + t.Fatalf("task prose was rewritten: %q", got) + } + if !strings.Contains(got, "data => inputs/data") { + t.Fatalf("task is missing the explicit input mapping: %q", got) + } +} + +func TestDelegationTaskWithInputAliasesRewritesOnlyExactPathTokens(t *testing.T) { + const reference = "/workspace/agent/.uploads/report.pdf" + task := "Read " + reference + ", but preserve " + reference + ".backup and prefix" + reference + inputs := []delegateInput{{ + relativePath: ".uploads/report.pdf", + taskReferences: []string{reference}, + }} + staged := []DelegationArtifact{{Path: "inputs/report.pdf"}} + + got := delegationTaskWithInputAliases(task, inputs, staged) + if !strings.Contains(got, "Read inputs/report.pdf,") { + t.Fatalf("exact path token was not rewritten: %q", got) + } + if !strings.Contains(got, reference+".backup") { + t.Fatalf("path substring suffix was rewritten: %q", got) + } + if !strings.Contains(got, "prefix"+reference) { + t.Fatalf("path substring prefix was rewritten: %q", got) + } +} + +func TestRedactDelegationArtifactTextHandlesSlashVariants(t *testing.T) { + exchange := &DelegationArtifactExchange{ + hostRoot: `C:\workspace\collaboration\delegations\123`, + } + text := `failed at C:/workspace/collaboration/delegations/123/inputs/file.txt` + got := redactDelegationArtifactText(text, exchange) + if strings.Contains(got, "C:/workspace") { + t.Fatalf("slash-normalized host path was not redacted: %q", got) + } + if !strings.Contains(got, "inputs/file.txt") { + t.Fatalf("logical input path missing after redaction: %q", got) + } +} + +func TestDelegateTool_RetainsFailedExchange(t *testing.T) { + workspace := t.TempDir() + managedWorkspace := t.TempDir() + if err := os.WriteFile(filepath.Join(workspace, "input.txt"), []byte("input"), 0600); err != nil { + t.Fatal(err) + } + + var captured DelegateRequest + tool := NewDelegateTool(noopAgentLink{}, noopAgentCRUD{}, nil, func(_ context.Context, req DelegateRequest) (DelegateResult, error) { + captured = req + return DelegateResult{}, errors.New("child run failed") + }) + tool.SetWorkspace(managedWorkspace) + t.Cleanup(tool.Close) + ctx := WithToolWorkspace(makeDelegateCtx(t), workspace) + result := tool.Execute(ctx, map[string]any{ + "agent_key": "child-agent", + "task": "fail after staging", + "mode": "sync", + "inputs": []any{"input.txt"}, + }) + if result == nil || !result.IsError { + t.Fatalf("delegate result = %#v, want error", result) + } + exchangeRoot := filepath.Join( + managedWorkspace, + "collaboration", + "delegations", + captured.DelegationID, + ) + if _, err := os.Stat(filepath.Join(exchangeRoot, "inputs", "input.txt")); err != nil { + t.Fatalf("failed exchange input not retained: %v", err) + } + if _, err := os.Stat(filepath.Join(workspace, ".delegations", captured.DelegationID)); !os.IsNotExist(err) { + t.Fatalf("failed exchange was published: %v", err) + } +} + +type delegateTraceCaptureStore struct { + store.TracingStore + mu sync.Mutex + spans []store.SpanData +} + +func (s *delegateTraceCaptureStore) BatchCreateSpans(_ context.Context, spans []store.SpanData) error { + s.mu.Lock() + defer s.mu.Unlock() + s.spans = append(s.spans, spans...) + return nil +} + +func (s *delegateTraceCaptureStore) BatchUpdateTraceAggregates(_ context.Context, _ uuid.UUID) error { + return nil +} + +func (s *delegateTraceCaptureStore) RecoverStaleRunningTraces(_ context.Context, _ time.Time) (int64, error) { + return 0, nil +} + +func (s *delegateTraceCaptureStore) DeleteTracesOlderThan(_ context.Context, _ time.Time) (int64, error) { + return 0, nil +} + +func TestDelegateTool_EmitsSanitizedArtifactLifecycleOnDelegateTrace(t *testing.T) { + workspace := t.TempDir() + traceStore := &delegateTraceCaptureStore{} + collector := tracing.NewCollector(traceStore) + collector.Start() + traceID := uuid.New() + + runFn := func(_ context.Context, req DelegateRequest) (DelegateResult, error) { + if err := os.WriteFile(filepath.Join(req.DelegateOutputsPath, "result.json"), []byte("{}"), 0600); err != nil { + return DelegateResult{}, err + } + return DelegateResult{Content: "done", TraceID: traceID}, nil + } + tool := newDelegateTestTool(t, noopAgentLink{}, runFn) + ctx := tracing.WithCollector(WithToolWorkspace(makeDelegateCtx(t), workspace), collector) + result := tool.Execute(ctx, map[string]any{ + "agent_key": "child-agent", + "task": "create result", + "mode": "sync", + }) + if result == nil || result.IsError { + t.Fatalf("delegate result = %#v, want success", result) + } + collector.Stop() + + traceStore.mu.Lock() + defer traceStore.mu.Unlock() + if len(traceStore.spans) != 3 { + t.Fatalf("spans = %#v, want staged, published, cleaned", traceStore.spans) + } + wantNames := []string{ + "delegate.artifacts.staged", + "delegate.artifacts.published", + "delegate.artifacts.cleaned", + } + for i, span := range traceStore.spans { + if span.TraceID != traceID || span.SpanType != store.SpanTypeEvent || + span.Name != wantNames[i] { + t.Fatalf("lifecycle span %d = %#v, want %s", i, span, wantNames[i]) + } + metadata := string(span.Metadata) + if strings.Contains(metadata, workspace) { + t.Fatalf("lifecycle metadata leaked workspace: %s", metadata) + } + } + for _, index := range []int{1, 2} { + metadata := string(traceStore.spans[index].Metadata) + if !strings.Contains(metadata, `"path":"outputs/result.json"`) { + t.Fatalf("output lifecycle metadata = %s", metadata) + } + } +} + +func TestDelegateTool_EmitsSanitizedFailedArtifactLifecycle(t *testing.T) { + workspace := t.TempDir() + if err := os.WriteFile(filepath.Join(workspace, "input.txt"), []byte("input"), 0600); err != nil { + t.Fatal(err) + } + traceStore := &delegateTraceCaptureStore{} + collector := tracing.NewCollector(traceStore) + collector.Start() + traceID := uuid.New() + + runFn := func(_ context.Context, req DelegateRequest) (DelegateResult, error) { + if req.OnTraceCreated != nil { + req.OnTraceCreated(traceID) + } + return DelegateResult{TraceID: traceID}, fmt.Errorf( + "failed while using %s", + req.DelegateOutputsPath, + ) + } + tool := newDelegateTestTool(t, noopAgentLink{}, runFn) + ctx := tracing.WithCollector(WithToolWorkspace(makeDelegateCtx(t), workspace), collector) + result := tool.Execute(ctx, map[string]any{ + "agent_key": "child-agent", + "task": "fail safely", + "mode": "sync", + "inputs": []any{"input.txt"}, + }) + if result == nil || !result.IsError { + t.Fatalf("delegate result = %#v, want failure", result) + } + if strings.Contains(result.ForLLM, workspace) { + t.Fatalf("delegate error leaked workspace: %s", result.ForLLM) + } + collector.Stop() + + traceStore.mu.Lock() + defer traceStore.mu.Unlock() + if len(traceStore.spans) != 2 { + t.Fatalf("spans = %#v, want staged and failed", traceStore.spans) + } + wantNames := []string{ + "delegate.artifacts.staged", + "delegate.artifacts.failed", + } + for i, span := range traceStore.spans { + if span.TraceID != traceID || span.Name != wantNames[i] { + t.Fatalf("failure lifecycle span %d = %#v", i, span) + } + metadata := string(span.Metadata) + if strings.Contains(metadata, workspace) || + !strings.Contains(metadata, `"path":"inputs/input.txt"`) { + t.Fatalf("failure lifecycle metadata = %s", metadata) + } + } +} + +func TestDelegateTool_EmitsCancelledArtifactLifecycle(t *testing.T) { + traceStore := &delegateTraceCaptureStore{} + collector := tracing.NewCollector(traceStore) + collector.Start() + traceID := uuid.New() + + runFn := func(_ context.Context, req DelegateRequest) (DelegateResult, error) { + if req.OnTraceCreated != nil { + req.OnTraceCreated(traceID) + } + return DelegateResult{TraceID: traceID}, context.Canceled + } + tool := newDelegateTestTool(t, noopAgentLink{}, runFn) + ctx := tracing.WithCollector(makeDelegateCtx(t), collector) + result := tool.Execute(ctx, map[string]any{ + "agent_key": "child-agent", + "task": "cancel safely", + "mode": "sync", + }) + if result == nil || !result.IsError { + t.Fatalf("delegate result = %#v, want cancellation", result) + } + collector.Stop() + + traceStore.mu.Lock() + defer traceStore.mu.Unlock() + if len(traceStore.spans) != 2 || + traceStore.spans[0].Name != "delegate.artifacts.staged" || + traceStore.spans[1].Name != "delegate.artifacts.cancelled" { + t.Fatalf("cancellation lifecycle spans = %#v", traceStore.spans) + } + for _, span := range traceStore.spans { + if span.TraceID != traceID { + t.Fatalf("cancellation trace ID = %s, want %s", span.TraceID, traceID) + } + } +} + +func TestDelegateTool_SweeperDeletesOnlyRegisteredExpiredExchange(t *testing.T) { + tenantWorkspace := t.TempDir() + delegationID := uuid.New() + exchangeRoot := filepath.Join(tenantWorkspace, "collaboration", "delegations", delegationID.String()) + if err := os.MkdirAll(exchangeRoot, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(exchangeRoot, "retained.txt"), []byte("retained"), 0600); err != nil { + t.Fatal(err) + } + outside := t.TempDir() + sentinel := filepath.Join(outside, "sentinel.txt") + if err := os.WriteFile(sentinel, []byte("keep"), 0600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(exchangeRoot, "hostile-link")); err != nil { + t.Logf("symlink cleanup coverage unavailable: %v", err) + } + tool := NewDelegateTool(noopAgentLink{}, noopAgentCRUD{}, nil, func(_ context.Context, _ DelegateRequest) (DelegateResult, error) { + return DelegateResult{}, nil + }) + item := retainedDelegationArtifact{ + tenantWorkspace: tenantWorkspace, + tenantID: store.MasterTenantID, + delegationID: delegationID, + retainUntil: time.Now().Add(-time.Minute), + } + tool.retained[retainedDelegationArtifactKey(tenantWorkspace, delegationID)] = item + tool.sweepRetainedDelegationExchanges(time.Now()) + + if _, err := os.Stat(exchangeRoot); !os.IsNotExist(err) { + t.Fatalf("expired exchange still exists: %v", err) + } + if len(tool.retained) != 0 { + t.Fatalf("retained registry = %#v, want empty", tool.retained) + } + if got, err := os.ReadFile(sentinel); err != nil || string(got) != "keep" { + t.Fatalf("secure cleanup followed hostile link: %q, %v", got, err) + } +} + +func TestDelegateTool_RecoversRetainedExchangeAfterRestart(t *testing.T) { + tenantWorkspace := t.TempDir() + callerWorkspace := filepath.Join(tenantWorkspace, "agents", "caller") + if err := os.MkdirAll(callerWorkspace, 0750); err != nil { + t.Fatal(err) + } + callerRoot, err := OpenDelegationArtifactRoot(callerWorkspace) + if err != nil { + t.Fatal(err) + } + delegationID := uuid.New() + exchange, err := NewDelegationArtifactExchange( + tenantWorkspace, + store.MasterTenantID, + delegationID, + DelegationArtifactLimits{}, + time.Minute, + ) + if err != nil { + t.Fatal(err) + } + + first := NewDelegateTool(noopAgentLink{}, noopAgentCRUD{}, nil, func(_ context.Context, _ DelegateRequest) (DelegateResult, error) { + return DelegateResult{}, nil + }) + first.SetWorkspace(tenantWorkspace) + job := &delegateArtifactJob{ + req: DelegateRequest{DelegationID: delegationID.String()}, + callerRoot: callerRoot, + callerWorkspace: callerWorkspace, + tenantWorkspace: tenantWorkspace, + tenantID: store.MasterTenantID, + delegationID: delegationID, + } + job.callerLocation = first.resolveDelegationCallerLocation(job) + exchange.RetainFailure(time.Now().Add(-2*time.Minute), "test_failure") + if err := first.registerRetainedDelegationExchange(exchange, job, artifactLifecycleFailed); err != nil { + t.Fatalf("register retained exchange: %v", err) + } + stateBytes, err := os.ReadFile(filepath.Join( + tenantWorkspace, + "collaboration", + "delegations", + delegationID.String(), + delegationArtifactLifecycleFile, + )) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(stateBytes), tenantWorkspace) || + strings.Contains(string(stateBytes), callerWorkspace) { + t.Fatalf("lifecycle state persisted a host path: %s", stateBytes) + } + if err := exchange.Close(); err != nil { + t.Fatal(err) + } + if err := callerRoot.Close(); err != nil { + t.Fatal(err) + } + first.Close() + + restarted := NewDelegateTool(noopAgentLink{}, noopAgentCRUD{}, nil, func(_ context.Context, _ DelegateRequest) (DelegateResult, error) { + return DelegateResult{}, nil + }) + restarted.SetWorkspace(tenantWorkspace) + if len(restarted.retained) != 1 { + t.Fatalf("recovered retained registry = %#v, want one", restarted.retained) + } + restarted.sweepRetainedDelegationExchanges(time.Now()) + restarted.Close() + + exchangeRoot := filepath.Join( + tenantWorkspace, + "collaboration", + "delegations", + delegationID.String(), + ) + if _, err := os.Stat(exchangeRoot); !os.IsNotExist(err) { + t.Fatalf("restart janitor left expired exchange: %v", err) + } +} + +func TestDelegateTool_RecoversStaleArtifactLifecycleStates(t *testing.T) { + testCases := []struct { + name string + status delegationArtifactLifecycleStatus + }{ + {name: "staging", status: artifactLifecycleStaging}, + {name: "running", status: artifactLifecycleRunning}, + {name: "publishing", status: artifactLifecyclePublishing}, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + tenantWorkspace := t.TempDir() + callerWorkspace := filepath.Join(tenantWorkspace, "agents", "caller") + if err := os.MkdirAll(callerWorkspace, 0750); err != nil { + t.Fatal(err) + } + callerRoot, err := OpenDelegationArtifactRoot(callerWorkspace) + if err != nil { + t.Fatal(err) + } + delegationID := uuid.New() + exchange, err := NewDelegationArtifactExchange( + tenantWorkspace, + store.MasterTenantID, + delegationID, + DelegationArtifactLimits{}, + time.Minute, + ) + if err != nil { + t.Fatal(err) + } + first := NewDelegateTool( + noopAgentLink{}, + noopAgentCRUD{}, + nil, + func(_ context.Context, _ DelegateRequest) (DelegateResult, error) { + return DelegateResult{}, nil + }, + ) + first.workspace = tenantWorkspace + job := &delegateArtifactJob{ + req: DelegateRequest{DelegationID: delegationID.String()}, + callerRoot: callerRoot, + callerWorkspace: callerWorkspace, + tenantWorkspace: tenantWorkspace, + tenantID: store.MasterTenantID, + delegationID: delegationID, + } + job.callerLocation = first.resolveDelegationCallerLocation(job) + if err := first.updateActiveDelegationLifecycle(exchange, job); err != nil { + t.Fatalf("update staging lifecycle: %v", err) + } + + publicationTempPath := "" + if testCase.status == artifactLifecycleRunning || + testCase.status == artifactLifecyclePublishing { + if err := first.markDelegationRunning(exchange, job, time.Now()); err != nil { + t.Fatalf("mark running: %v", err) + } + } + if testCase.status == artifactLifecyclePublishing { + publicationTempPath = filepath.ToSlash(filepath.Join( + ".delegations", + ".tmp-"+delegationID.String()+"-"+uuid.NewString(), + )) + if err := first.markDelegationPublishing( + exchange, + job, + publicationTempPath, + time.Now(), + ); err != nil { + t.Fatalf("mark publishing: %v", err) + } + if err := os.MkdirAll( + filepath.Join(callerWorkspace, filepath.FromSlash(publicationTempPath)), + 0700, + ); err != nil { + t.Fatal(err) + } + } + + finalSentinel := filepath.Join( + callerWorkspace, + ".delegations", + delegationID.String(), + "keep.txt", + ) + if err := os.MkdirAll(filepath.Dir(finalSentinel), 0750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(finalSentinel, []byte("keep"), 0600); err != nil { + t.Fatal(err) + } + if err := exchange.Close(); err != nil { + t.Fatal(err) + } + if err := callerRoot.Close(); err != nil { + t.Fatal(err) + } + first.Close() + + restarted := NewDelegateTool( + noopAgentLink{}, + noopAgentCRUD{}, + nil, + func(_ context.Context, _ DelegateRequest) (DelegateResult, error) { + return DelegateResult{}, nil + }, + ) + recoveredAfter := time.Now().UTC() + restarted.SetWorkspace(tenantWorkspace) + defer restarted.Close() + if len(restarted.retained) != 1 { + t.Fatalf("recovered registry = %#v, want one", restarted.retained) + } + + lifecyclePath := filepath.Join( + tenantWorkspace, + "collaboration", + "delegations", + delegationID.String(), + delegationArtifactLifecycleFile, + ) + stateBytes, err := os.ReadFile(lifecyclePath) + if err != nil { + t.Fatal(err) + } + var state delegationArtifactLifecycleState + if err := json.Unmarshal(stateBytes, &state); err != nil { + t.Fatal(err) + } + if state.Status != artifactLifecycleFailed || + state.FailedAt == nil || + state.ReasonCode != "artifact_recovered_stale" { + t.Fatalf("recovered state = %#v", state) + } + earliestExpiry := recoveredAfter.Add(delegationArtifactFailureTTL) + if state.RetainUntil.Before(earliestExpiry) || + state.RetainUntil.After(time.Now().UTC().Add(delegationArtifactFailureTTL)) { + t.Fatalf("retain until = %v, want recovery time + %v", state.RetainUntil, delegationArtifactFailureTTL) + } + + restarted.sweepRetainedDelegationExchanges(state.RetainUntil.Add(-time.Nanosecond)) + if _, err := os.Stat(lifecyclePath); err != nil { + t.Fatalf("retained exchange removed before TTL: %v", err) + } + restarted.sweepRetainedDelegationExchanges(state.RetainUntil) + if _, err := os.Stat(filepath.Dir(lifecyclePath)); !os.IsNotExist(err) { + t.Fatalf("expired exchange remains: %v", err) + } + if publicationTempPath != "" { + if _, err := os.Stat(filepath.Join( + callerWorkspace, + filepath.FromSlash(publicationTempPath), + )); !os.IsNotExist(err) { + t.Fatalf("recorded publication temp remains: %v", err) + } + } + if got, err := os.ReadFile(finalSentinel); err != nil || string(got) != "keep" { + t.Fatalf("final UUID publication directory was modified: %q, %v", got, err) + } + }) + } +} + +func TestDelegateTool_RecoveryPromotesDurablePublishingState(t *testing.T) { + tenantWorkspace := t.TempDir() + callerWorkspace := filepath.Join(tenantWorkspace, "agents", "caller") + if err := os.MkdirAll(callerWorkspace, 0o750); err != nil { + t.Fatal(err) + } + callerRoot, err := OpenDelegationArtifactRoot(callerWorkspace) + if err != nil { + t.Fatal(err) + } + delegationID := uuid.New() + exchange, err := NewDelegationArtifactExchange( + tenantWorkspace, + store.MasterTenantID, + delegationID, + DelegationArtifactLimits{}, + time.Minute, + ) + if err != nil { + t.Fatal(err) + } + first := NewDelegateTool( + noopAgentLink{}, + noopAgentCRUD{}, + nil, + func(_ context.Context, _ DelegateRequest) (DelegateResult, error) { + return DelegateResult{}, nil + }, + ) + first.workspace = tenantWorkspace + job := &delegateArtifactJob{ + req: DelegateRequest{DelegationID: delegationID.String()}, + callerRoot: callerRoot, + callerWorkspace: callerWorkspace, + tenantWorkspace: tenantWorkspace, + tenantID: store.MasterTenantID, + delegationID: delegationID, + } + job.callerLocation = first.resolveDelegationCallerLocation(job) + if err := first.updateActiveDelegationLifecycle(exchange, job); err != nil { + t.Fatalf("update lifecycle: %v", err) + } + if err := first.markDelegationRunning(exchange, job, time.Now()); err != nil { + t.Fatalf("mark running: %v", err) + } + if err := os.WriteFile( + filepath.Join(exchange.OutputsHostPath(), "result.txt"), + []byte("durable result"), + 0o600, + ); err != nil { + t.Fatal(err) + } + publication, err := exchange.publishWithPreparation( + context.Background(), + callerRoot, + time.Now(), + func(tempPath string) error { + return first.markDelegationPublishing(exchange, job, tempPath, time.Now()) + }, + ) + if err != nil { + t.Fatalf("publish before simulated crash: %v", err) + } + if err := exchange.Close(); err != nil { + t.Fatal(err) + } + if err := callerRoot.Close(); err != nil { + t.Fatal(err) + } + first.Close() + + restarted := NewDelegateTool( + noopAgentLink{}, + noopAgentCRUD{}, + nil, + func(_ context.Context, _ DelegateRequest) (DelegateResult, error) { + return DelegateResult{}, nil + }, + ) + restarted.SetWorkspace(tenantWorkspace) + defer restarted.Close() + + item, ok := restarted.retained[retainedDelegationArtifactKey(tenantWorkspace, delegationID)] + if !ok || !item.publicationDurable || !item.retainUntil.IsZero() { + t.Fatalf("recovered publication = %#v, want durable immediate cleanup", item) + } + lifecyclePath := filepath.Join( + tenantWorkspace, + "collaboration", + "delegations", + delegationID.String(), + delegationArtifactLifecycleFile, + ) + stateBytes, err := os.ReadFile(lifecyclePath) + if err != nil { + t.Fatal(err) + } + var state delegationArtifactLifecycleState + if err := json.Unmarshal(stateBytes, &state); err != nil { + t.Fatal(err) + } + if state.Status != artifactLifecyclePublished || state.ReasonCode != "artifact_published" { + t.Fatalf("recovered lifecycle = %#v, want published", state) + } + + restarted.sweepRetainedDelegationExchanges(time.Now()) + if _, err := os.Stat(filepath.Dir(lifecyclePath)); !os.IsNotExist(err) { + t.Fatalf("published exchange remains after cleanup: %v", err) + } + publishedOutput := filepath.Join( + callerWorkspace, + filepath.FromSlash(publication.RootPath), + "outputs", + "result.txt", + ) + if got, err := os.ReadFile(publishedOutput); err != nil || string(got) != "durable result" { + t.Fatalf("durable publication changed during recovery: %q, %v", got, err) + } +} + +func TestDelegateTool_RecoveryCleansInvalidAndExpiresCorruptExchanges(t *testing.T) { + tenantWorkspace := t.TempDir() + delegationsRoot := filepath.Join(tenantWorkspace, "collaboration", "delegations") + invalidRoot := filepath.Join(delegationsRoot, "not-a-delegation") + if err := os.MkdirAll(invalidRoot, 0750); err != nil { + t.Fatal(err) + } + corruptID := uuid.New() + corruptRoot := filepath.Join(delegationsRoot, corruptID.String()) + if err := os.MkdirAll(corruptRoot, 0750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile( + filepath.Join(corruptRoot, delegationArtifactLifecycleFile), + []byte("{not-json"), + 0600, + ); err != nil { + t.Fatal(err) + } + + tool := NewDelegateTool( + noopAgentLink{}, + noopAgentCRUD{}, + nil, + func(_ context.Context, _ DelegateRequest) (DelegateResult, error) { + return DelegateResult{}, nil + }, + ) + tool.SetWorkspace(tenantWorkspace) + defer tool.Close() + + if _, err := os.Stat(invalidRoot); !os.IsNotExist(err) { + t.Fatalf("invalid exchange entry remains after recovery: %v", err) + } + item, ok := tool.retained[retainedDelegationArtifactKey(tenantWorkspace, corruptID)] + if !ok { + t.Fatalf("corrupt exchange was not registered for bounded retention: %#v", tool.retained) + } + tool.sweepRetainedDelegationExchanges(item.retainUntil.Add(-time.Nanosecond)) + if _, err := os.Stat(corruptRoot); err != nil { + t.Fatalf("corrupt exchange removed before retention elapsed: %v", err) + } + tool.sweepRetainedDelegationExchanges(item.retainUntil) + if _, err := os.Stat(corruptRoot); !os.IsNotExist(err) { + t.Fatalf("corrupt exchange remains after retention elapsed: %v", err) + } +} + +func TestDelegateTool_RecoversPublicationCleanupAfterRestart(t *testing.T) { + tenantWorkspace := t.TempDir() + callerWorkspace := filepath.Join(tenantWorkspace, "agents", "caller") + if err := os.MkdirAll(callerWorkspace, 0750); err != nil { + t.Fatal(err) + } + callerRoot, err := OpenDelegationArtifactRoot(callerWorkspace) + if err != nil { + t.Fatal(err) + } + delegationID := uuid.New() + exchange, err := NewDelegationArtifactExchange( + tenantWorkspace, + store.MasterTenantID, + delegationID, + DelegationArtifactLimits{}, + time.Minute, + ) + if err != nil { + t.Fatal(err) + } + + first := NewDelegateTool( + noopAgentLink{}, + noopAgentCRUD{}, + nil, + func(_ context.Context, _ DelegateRequest) (DelegateResult, error) { + return DelegateResult{}, nil + }, + ) + first.workspace = tenantWorkspace + job := &delegateArtifactJob{ + req: DelegateRequest{DelegationID: delegationID.String()}, + callerRoot: callerRoot, + callerWorkspace: callerWorkspace, + tenantWorkspace: tenantWorkspace, + tenantID: store.MasterTenantID, + delegationID: delegationID, + } + job.callerLocation = first.resolveDelegationCallerLocation(job) + if err := first.updateActiveDelegationLifecycle(exchange, job); err != nil { + t.Fatalf("update staging lifecycle: %v", err) + } + if err := first.markDelegationRunning(exchange, job, time.Now()); err != nil { + t.Fatalf("mark running: %v", err) + } + if err := os.WriteFile( + filepath.Join(exchange.OutputsHostPath(), "result.txt"), + []byte("durable"), + 0600, + ); err != nil { + t.Fatal(err) + } + var publicationTempPath string + publication, err := exchange.publishWithPreparation( + context.Background(), + callerRoot, + time.Now(), + func(tempPath string) error { + publicationTempPath = tempPath + return first.markDelegationPublishing(exchange, job, tempPath, time.Now()) + }, + ) + if err != nil { + t.Fatalf("publish before simulated restart: %v", err) + } + if err := first.markDelegationPublished(exchange, job, publication.Manifest.PublishedAt); err != nil { + t.Fatalf("mark published: %v", err) + } + if publicationTempPath == "" { + t.Fatal("publication temp path was not recorded") + } + if err := exchange.Close(); err != nil { + t.Fatal(err) + } + if err := callerRoot.Close(); err != nil { + t.Fatal(err) + } + first.Close() + + restarted := NewDelegateTool( + noopAgentLink{}, + noopAgentCRUD{}, + nil, + func(_ context.Context, _ DelegateRequest) (DelegateResult, error) { + return DelegateResult{}, nil + }, + ) + restarted.SetWorkspace(tenantWorkspace) + if len(restarted.retained) != 1 { + t.Fatalf("recovered cleanup registry = %#v, want one", restarted.retained) + } + restarted.sweepRetainedDelegationExchanges(time.Now()) + restarted.Close() + + exchangeRoot := filepath.Join( + tenantWorkspace, + "collaboration", + "delegations", + delegationID.String(), + ) + if _, err := os.Stat(exchangeRoot); !os.IsNotExist(err) { + t.Fatalf("restart janitor left publication exchange: %v", err) + } + publicationRoot := filepath.Join( + callerWorkspace, + ".delegations", + delegationID.String(), + ) + published, err := os.ReadFile(filepath.Join(publicationRoot, "outputs", "result.txt")) + if err != nil || string(published) != "durable" { + t.Fatalf("restart janitor damaged durable publication: %q, %v", published, err) + } +} + +func TestDelegateTool_RetriesPublishedExchangeCleanup(t *testing.T) { + managedWorkspace := t.TempDir() + callerWorkspace := t.TempDir() + traceStore := &delegateTraceCaptureStore{} + collector := tracing.NewCollector(traceStore) + collector.Start() + traceID := uuid.New() + var captured DelegateRequest + tool := NewDelegateTool( + noopAgentLink{}, + noopAgentCRUD{}, + nil, + func(_ context.Context, req DelegateRequest) (DelegateResult, error) { + captured = req + if err := os.WriteFile( + filepath.Join(req.DelegateOutputsPath, "result.txt"), + []byte("durable"), + 0600, + ); err != nil { + return DelegateResult{}, err + } + return DelegateResult{Content: "done", TraceID: traceID}, nil + }, + ) + tool.SetWorkspace(managedWorkspace) + defer tool.Close() + + removeCalls := 0 + tool.removeExchange = func(tenantWorkspace string, delegationID uuid.UUID) error { + removeCalls++ + if removeCalls == 1 { + return errors.New("injected cleanup failure") + } + return tool.removeDelegationExchange(tenantWorkspace, delegationID) + } + ctx := tracing.WithCollector( + WithToolWorkspace(makeDelegateCtx(t), callerWorkspace), + collector, + ) + result := tool.Execute(ctx, map[string]any{ + "agent_key": "child-agent", + "task": "publish output", + "mode": "sync", + }) + if result == nil || result.IsError { + t.Fatalf("delegate result = %#v, want success", result) + } + delegationID := uuid.MustParse(captured.DelegationID) + exchangeRoot := filepath.Join( + managedWorkspace, + "collaboration", + "delegations", + delegationID.String(), + ) + if _, err := os.Stat(exchangeRoot); err != nil { + t.Fatalf("failed cleanup did not retain exchange for retry: %v", err) + } + tool.retainedMu.Lock() + retainedCount := len(tool.retained) + sweeperStarted := tool.sweeperStarted + tool.retainedMu.Unlock() + if retainedCount != 1 || !sweeperStarted { + t.Fatalf("automatic cleanup registry = %d, sweeper started = %v", retainedCount, sweeperStarted) + } + + tool.sweepRetainedDelegationExchanges(time.Now()) + if removeCalls != 2 { + t.Fatalf("cleanup calls = %d, want immediate attempt plus retry", removeCalls) + } + if _, err := os.Stat(exchangeRoot); !os.IsNotExist(err) { + t.Fatalf("cleanup retry left exchange: %v", err) + } + published, err := os.ReadFile(filepath.Join( + callerWorkspace, + ".delegations", + delegationID.String(), + "outputs", + "result.txt", + )) + if err != nil || string(published) != "durable" { + t.Fatalf("cleanup retry damaged durable output: %q, %v", published, err) + } + collector.Stop() + traceStore.mu.Lock() + defer traceStore.mu.Unlock() + if len(traceStore.spans) != 3 || + traceStore.spans[2].TraceID != traceID || + traceStore.spans[2].Name != "delegate.artifacts.cleaned" { + t.Fatalf("cleanup retry spans = %#v, want cleaned on original trace", traceStore.spans) + } +} diff --git a/internal/tools/delegation_artifact_janitor.go b/internal/tools/delegation_artifact_janitor.go new file mode 100644 index 00000000..7184ec7f --- /dev/null +++ b/internal/tools/delegation_artifact_janitor.go @@ -0,0 +1,708 @@ +package tools + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "os" + "path" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/google/uuid" + + "github.com/nextlevelbuilder/goclaw/internal/config" + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +const delegationArtifactManifestMaxBytes = 1024 * 1024 + +type recoveredPublicationStatus uint8 + +const ( + recoveredPublicationMissing recoveredPublicationStatus = iota + recoveredPublicationInvalid + recoveredPublicationValid +) + +func isArtifactSymlinkError(err error) bool { + return errors.Is(err, errArtifactSymlink) || errors.Is(err, errArtifactReparsePoint) +} + +type retainedDelegationArtifact struct { + tenantWorkspace string + tenantID uuid.UUID + tenantSlug string + delegationID uuid.UUID + retainUntil time.Time + callerLocation *delegationArtifactCallerLocation + publicationTempPath string + publicationDurable bool + onCleaned func() +} + +func (t *DelegateTool) resolveDelegationCallerLocation( + job *delegateArtifactJob, +) *delegationArtifactCallerLocation { + type candidate struct { + name string + root string + } + candidates := []candidate{{name: "workspace", root: job.tenantWorkspace}} + if t.dataDir != "" { + candidates = append(candidates, candidate{ + name: "data", + root: config.TenantDataDir(t.dataDir, job.tenantID, job.tenantSlug), + }) + } + for _, candidate := range candidates { + relativePath, ok := artifactRelativeToRoot(candidate.root, job.callerWorkspace) + if !ok { + continue + } + return &delegationArtifactCallerLocation{ + Base: candidate.name, + RelativePath: relativePath, + } + } + return nil +} + +func artifactRelativeToRoot(root, target string) (string, bool) { + if root == "" || target == "" { + return "", false + } + absoluteRoot, err := filepath.Abs(root) + if err != nil { + return "", false + } + absoluteTarget, err := filepath.Abs(target) + if err != nil { + return "", false + } + relativePath, err := filepath.Rel(absoluteRoot, absoluteTarget) + if err != nil || relativePath == ".." || + strings.HasPrefix(relativePath, ".."+string(filepath.Separator)) { + return "", false + } + relativePath = filepath.ToSlash(relativePath) + if relativePath == "." { + return relativePath, true + } + validated, err := validateArtifactRelativePath(relativePath) + return validated, err == nil +} + +func (t *DelegateTool) updateActiveDelegationLifecycle( + exchange *DelegationArtifactExchange, + job *delegateArtifactJob, +) error { + state, err := readDelegationArtifactLifecycleState(exchange.root, job.delegationID) + if err != nil { + return err + } + if state.Status != artifactLifecycleStaging { + return ErrArtifactState + } + state.TenantSlug = job.tenantSlug + state.CallerLocation = job.callerLocation + return exchange.persistLifecycleState(state) +} + +func (t *DelegateTool) markDelegationRunning( + exchange *DelegationArtifactExchange, + job *delegateArtifactJob, + now time.Time, +) error { + state, err := readDelegationArtifactLifecycleState(exchange.root, job.delegationID) + if err != nil { + return err + } + if state.Status != artifactLifecycleStaging { + return ErrArtifactState + } + state.TenantSlug = job.tenantSlug + state.Status = artifactLifecycleRunning + state.FailedAt = nil + state.RetainUntil = now.UTC().Add(delegationArtifactFailureTTL) + state.ReasonCode = "artifact_running" + state.CallerLocation = job.callerLocation + state.PublicationTempPath = "" + return exchange.persistLifecycleState(state) +} + +func (t *DelegateTool) markDelegationPublishing( + exchange *DelegationArtifactExchange, + job *delegateArtifactJob, + tempPath string, + now time.Time, +) error { + if err := validateDelegationPublicationTempPath(job.delegationID, tempPath); err != nil { + return err + } + state, err := readDelegationArtifactLifecycleState(exchange.root, job.delegationID) + if err != nil { + return err + } + if state.Status != artifactLifecycleRunning { + return ErrArtifactState + } + state.TenantSlug = job.tenantSlug + state.Status = artifactLifecyclePublishing + state.FailedAt = nil + state.RetainUntil = now.UTC().Add(delegationArtifactFailureTTL) + state.ReasonCode = "artifact_publishing" + state.CallerLocation = job.callerLocation + state.PublicationTempPath = tempPath + return exchange.persistLifecycleState(state) +} + +func (t *DelegateTool) markDelegationPublished( + exchange *DelegationArtifactExchange, + job *delegateArtifactJob, + now time.Time, +) error { + state, err := readDelegationArtifactLifecycleState(exchange.root, job.delegationID) + if err != nil { + return err + } + if state.Status != artifactLifecyclePublishing { + return ErrArtifactState + } + state.Status = artifactLifecyclePublished + state.FailedAt = nil + state.RetainUntil = now.UTC() + state.ReasonCode = "artifact_published" + return exchange.persistLifecycleState(state) +} + +func (t *DelegateTool) registerRetainedDelegationExchange( + exchange *DelegationArtifactExchange, + job *delegateArtifactJob, + status delegationArtifactLifecycleStatus, +) error { + if exchange == nil || job == nil { + return ErrArtifactState + } + + retention, ok := exchange.FailureRetention() + if !ok { + return ErrArtifactState + } + state, err := readDelegationArtifactLifecycleState(exchange.root, job.delegationID) + if err != nil { + return err + } + failedAt := retention.FailedAt.UTC() + state.TenantSlug = job.tenantSlug + if status != artifactLifecycleCancelled { + status = artifactLifecycleFailed + } + state.Status = status + state.FailedAt = &failedAt + state.RetainUntil = retention.RetainUntil.UTC() + state.ReasonCode = retention.ReasonCode + state.CallerLocation = job.callerLocation + if err := exchange.persistLifecycleState(state); err != nil { + return err + } + + t.addRetainedDelegationArtifact(retainedDelegationArtifact{ + tenantWorkspace: job.tenantWorkspace, + tenantID: job.tenantID, + tenantSlug: job.tenantSlug, + delegationID: job.delegationID, + retainUntil: retention.RetainUntil, + callerLocation: job.callerLocation, + publicationTempPath: state.PublicationTempPath, + }) + return nil +} + +func (t *DelegateTool) registerPublishedDelegationCleanup( + job *delegateArtifactJob, + publicationTempPath string, + onCleaned func(), +) { + t.addRetainedDelegationArtifact(retainedDelegationArtifact{ + tenantWorkspace: job.tenantWorkspace, + tenantID: job.tenantID, + tenantSlug: job.tenantSlug, + delegationID: job.delegationID, + retainUntil: time.Time{}, + callerLocation: job.callerLocation, + publicationTempPath: publicationTempPath, + publicationDurable: true, + onCleaned: onCleaned, + }) +} + +func (t *DelegateTool) addRetainedDelegationArtifact(item retainedDelegationArtifact) { + key := retainedDelegationArtifactKey(item.tenantWorkspace, item.delegationID) + t.retainedMu.Lock() + defer t.retainedMu.Unlock() + if t.sweeperClosed { + return + } + if existing, ok := t.retained[key]; ok && item.onCleaned == nil { + item.onCleaned = existing.onCleaned + } + t.retained[key] = item + if t.sweeperStarted { + return + } + t.sweeperStarted = true + go t.runDelegationArtifactSweeper() +} + +func retainedDelegationArtifactKey(tenantWorkspace string, delegationID uuid.UUID) string { + return tenantWorkspace + "\x00" + delegationID.String() +} + +func (t *DelegateTool) runDelegationArtifactSweeper() { + ticker := time.NewTicker(delegationArtifactSweepInterval) + defer func() { + ticker.Stop() + close(t.sweeperDone) + }() + for { + select { + case now := <-ticker.C: + t.recoverRetainedDelegationExchanges() + t.sweepRetainedDelegationExchanges(now) + case <-t.sweeperStop: + return + } + } +} + +func (t *DelegateTool) recoverRetainedDelegationExchanges() { + for _, tenantWorkspace := range t.delegationTenantWorkspaces() { + if err := t.recoverTenantDelegationExchanges(tenantWorkspace); err != nil { + slog.Warn("delegate.artifact_recovery_scan_failed") + } + } +} + +func (t *DelegateTool) delegationTenantWorkspaces() []string { + if t.workspace == "" { + return nil + } + roots := make([]string, 0, 8) + if info, err := os.Lstat(t.workspace); err == nil && info.IsDir() && info.Mode()&os.ModeSymlink == 0 { + roots = append(roots, t.workspace) + } + tenantParent := filepath.Join(t.workspace, "tenants") + entries, err := os.ReadDir(tenantParent) + if err != nil { + return roots + } + for _, entry := range entries { + if entry.Type()&os.ModeSymlink != 0 { + continue + } + info, err := entry.Info() + if err != nil || !info.IsDir() { + continue + } + roots = append(roots, filepath.Join(tenantParent, entry.Name())) + } + sort.Strings(roots) + return roots +} + +func (t *DelegateTool) recoverTenantDelegationExchanges(tenantWorkspace string) error { + tenantRoot, err := openArtifactSecureRoot(tenantWorkspace) + if err != nil { + return err + } + defer tenantRoot.close() + delegationsRoot, err := tenantRoot.openSubroot("collaboration/delegations") + if err != nil { + if isArtifactNotExist(err) { + return nil + } + return err + } + defer delegationsRoot.close() + names, err := tenantRoot.readDir("collaboration/delegations") + if err != nil { + return err + } + sort.Strings(names) + recoveredAt := time.Now().UTC() + for _, name := range names { + delegationID, err := uuid.Parse(name) + if err != nil || delegationID == uuid.Nil || delegationID.String() != name { + if cleanupErr := delegationsRoot.removeTree(name); cleanupErr != nil { + slog.Warn("delegate.artifact_invalid_exchange_cleanup_failed", + "entry", name, + "error", cleanupErr, + ) + } else { + slog.Warn("security.delegation_artifact_invalid_exchange_removed", + "entry", name, + ) + } + continue + } + exchangeRoot, err := delegationsRoot.openSubroot(name) + if err != nil { + slog.Warn("delegate.artifact_exchange_recovery_open_failed", + "delegation_id", delegationID, + "error", err, + ) + continue + } + state, stateErr := readDelegationArtifactLifecycleState(exchangeRoot, delegationID) + if stateErr != nil { + _ = exchangeRoot.close() + t.addRetainedDelegationArtifact(retainedDelegationArtifact{ + tenantWorkspace: tenantWorkspace, + delegationID: delegationID, + retainUntil: recoveredAt.Add(delegationArtifactFailureTTL), + }) + slog.Warn("delegate.artifact_corrupt_exchange_retained", + "delegation_id", delegationID, + "error", stateErr, + ) + continue + } + tenantID, err := uuid.Parse(state.TenantID) + if err != nil || tenantID == uuid.Nil { + _ = exchangeRoot.close() + continue + } + if !t.lifecycleStateMatchesTenantWorkspace(state, tenantWorkspace, tenantID) { + _ = exchangeRoot.close() + continue + } + retainUntil := state.RetainUntil + publicationDurable := false + switch state.Status { + case artifactLifecyclePublishing: + item := retainedDelegationArtifact{ + tenantWorkspace: tenantWorkspace, + tenantID: tenantID, + tenantSlug: state.TenantSlug, + delegationID: delegationID, + retainUntil: state.RetainUntil, + callerLocation: state.CallerLocation, + publicationTempPath: state.PublicationTempPath, + } + publicationStatus, inspectErr := t.inspectRecoveredPublication(item) + if inspectErr != nil { + _ = exchangeRoot.close() + t.addRetainedDelegationArtifact(item) + slog.Warn("delegate.artifact_publication_recovery_deferred", + "delegation_id", delegationID, + "error", inspectErr, + ) + continue + } + if publicationStatus == recoveredPublicationValid { + state.Status = artifactLifecyclePublished + state.FailedAt = nil + state.RetainUntil = recoveredAt + state.ReasonCode = "artifact_published" + retainUntil = time.Time{} + publicationDurable = true + if err := persistDelegationArtifactLifecycleState(exchangeRoot, delegationID, state); err != nil { + slog.Warn("delegate.artifact_publication_recovery_state_failed", + "delegation_id", delegationID, + "error", err, + ) + } + break + } + fallthrough + case artifactLifecycleStaging, artifactLifecycleRunning: + failedAt := recoveredAt + state.Status = artifactLifecycleFailed + state.FailedAt = &failedAt + state.RetainUntil = recoveredAt.Add(delegationArtifactFailureTTL) + state.ReasonCode = "artifact_recovered_stale" + if err := persistDelegationArtifactLifecycleState(exchangeRoot, delegationID, state); err != nil { + _ = exchangeRoot.close() + continue + } + retainUntil = state.RetainUntil + case artifactLifecyclePublished: + retainUntil = time.Time{} + publicationDurable = true + } + _ = exchangeRoot.close() + t.addRetainedDelegationArtifact(retainedDelegationArtifact{ + tenantWorkspace: tenantWorkspace, + tenantID: tenantID, + tenantSlug: state.TenantSlug, + delegationID: delegationID, + retainUntil: retainUntil, + callerLocation: state.CallerLocation, + publicationTempPath: state.PublicationTempPath, + publicationDurable: publicationDurable, + }) + } + return nil +} + +func (t *DelegateTool) inspectRecoveredPublication( + item retainedDelegationArtifact, +) (recoveredPublicationStatus, error) { + callerRootPath, err := t.resolveRetainedCallerRoot(item) + if err != nil { + return recoveredPublicationInvalid, nil + } + callerRoot, err := openArtifactSecureRoot(callerRootPath) + if err != nil { + if isArtifactNotExist(err) { + return recoveredPublicationMissing, nil + } + return recoveredPublicationMissing, err + } + defer callerRoot.close() + + finalPath := path.Join(".delegations", item.delegationID.String()) + publicationRoot, err := callerRoot.openSubroot(finalPath) + if err != nil { + switch { + case isArtifactNotExist(err): + return recoveredPublicationMissing, nil + case isArtifactSymlinkError(err): + return recoveredPublicationInvalid, nil + default: + return recoveredPublicationMissing, err + } + } + defer publicationRoot.close() + + entry, err := publicationRoot.openEntry("manifest.json") + if err != nil { + if isArtifactNotExist(err) || isArtifactSymlinkError(err) { + return recoveredPublicationInvalid, nil + } + return recoveredPublicationMissing, err + } + defer entry.close() + if entry.kind != artifactEntryRegular || entry.links != 1 || + entry.size <= 0 || entry.size > delegationArtifactManifestMaxBytes { + return recoveredPublicationInvalid, nil + } + encoded, err := io.ReadAll(io.LimitReader(entry.file, delegationArtifactManifestMaxBytes+1)) + if err != nil { + return recoveredPublicationMissing, err + } + if len(encoded) > delegationArtifactManifestMaxBytes { + return recoveredPublicationInvalid, nil + } + var manifest DelegationArtifactManifest + if err := json.Unmarshal(encoded, &manifest); err != nil { + return recoveredPublicationInvalid, nil + } + if manifest.SchemaVersion != DelegationArtifactManifestVersion || + manifest.DelegationID != item.delegationID.String() || + manifest.OutputCount != len(manifest.Outputs) || + manifest.OutputCount < 0 || + manifest.OutputCount > DelegationArtifactMaxFiles || + manifest.OutputBytes < 0 || + manifest.OutputBytes > DelegationArtifactMaxTotalBytes { + return recoveredPublicationInvalid, nil + } + + seen := make(map[string]struct{}, len(manifest.Outputs)) + var totalBytes int64 + for _, output := range manifest.Outputs { + if err := validateManifestOutputPath(output.Path); err != nil || + output.SizeBytes < 0 || + output.SizeBytes > DelegationArtifactMaxFileBytes || + output.SHA256 == "" { + return recoveredPublicationInvalid, nil + } + if _, exists := seen[output.Path]; exists { + return recoveredPublicationInvalid, nil + } + seen[output.Path] = struct{}{} + outputEntry, err := publicationRoot.openEntry(output.Path) + if err != nil { + if isArtifactNotExist(err) || isArtifactSymlinkError(err) { + return recoveredPublicationInvalid, nil + } + return recoveredPublicationMissing, err + } + valid := outputEntry.kind == artifactEntryRegular && + outputEntry.links == 1 && + outputEntry.size == output.SizeBytes + _ = outputEntry.close() + if !valid { + return recoveredPublicationInvalid, nil + } + totalBytes += output.SizeBytes + if totalBytes > DelegationArtifactMaxTotalBytes { + return recoveredPublicationInvalid, nil + } + } + if totalBytes != manifest.OutputBytes { + return recoveredPublicationInvalid, nil + } + return recoveredPublicationValid, nil +} + +func (t *DelegateTool) lifecycleStateMatchesTenantWorkspace( + state delegationArtifactLifecycleState, + tenantWorkspace string, + tenantID uuid.UUID, +) bool { + if tenantID == store.MasterTenantID { + expected, err := filepath.Abs(t.workspace) + if err != nil { + return false + } + actual, err := filepath.Abs(tenantWorkspace) + return err == nil && actual == expected + } + if state.TenantSlug == "" { + // A staging state can be created immediately before its tenant slug is + // added. It has no caller publication path and can only delete its scanned + // exchange directory. + return state.Status == artifactLifecycleStaging && + state.CallerLocation == nil && + state.PublicationTempPath == "" + } + expected := config.TenantWorkspace(t.workspace, tenantID, state.TenantSlug) + expectedAbs, err := filepath.Abs(expected) + if err != nil { + return false + } + actualAbs, err := filepath.Abs(tenantWorkspace) + return err == nil && actualAbs == expectedAbs +} + +func (t *DelegateTool) sweepRetainedDelegationExchanges(now time.Time) { + due := make([]retainedDelegationArtifact, 0, delegationArtifactSweepBatch) + t.retainedMu.Lock() + for _, item := range t.retained { + if !now.Before(item.retainUntil) { + due = append(due, item) + if len(due) == delegationArtifactSweepBatch { + break + } + } + } + t.retainedMu.Unlock() + + for _, item := range due { + if err := t.cleanupRetainedDelegationArtifact(item); err != nil { + slog.Warn("delegate.artifact_retention_cleanup_failed") + continue + } + t.retainedMu.Lock() + delete(t.retained, retainedDelegationArtifactKey(item.tenantWorkspace, item.delegationID)) + t.retainedMu.Unlock() + if item.onCleaned != nil { + item.onCleaned() + } + } +} + +func (t *DelegateTool) cleanupRetainedDelegationArtifact(item retainedDelegationArtifact) error { + if item.publicationTempPath != "" && !item.publicationDurable { + if item.callerLocation == nil { + return ErrArtifactState + } + if err := validateDelegationPublicationTempPath(item.delegationID, item.publicationTempPath); err != nil { + return err + } + callerRootPath, err := t.resolveRetainedCallerRoot(item) + if err != nil { + return err + } + callerRoot, err := openArtifactSecureRoot(callerRootPath) + if err != nil { + return err + } + err = callerRoot.removeTree(item.publicationTempPath) + _ = callerRoot.close() + if err != nil && !isArtifactNotExist(err) { + return err + } + } + return t.tryRemoveDelegationExchange(item.tenantWorkspace, item.delegationID) +} + +func (t *DelegateTool) resolveRetainedCallerRoot(item retainedDelegationArtifact) (string, error) { + if item.callerLocation == nil { + return "", ErrArtifactState + } + var base string + switch item.callerLocation.Base { + case "workspace": + base = item.tenantWorkspace + case "data": + if t.dataDir == "" { + return "", ErrArtifactState + } + base = config.TenantDataDir(t.dataDir, item.tenantID, item.tenantSlug) + default: + return "", ErrArtifactState + } + relativePath := item.callerLocation.RelativePath + if relativePath == "." { + return base, nil + } + validated, err := validateArtifactRelativePath(relativePath) + if err != nil { + return "", err + } + return filepath.Join(base, filepath.FromSlash(validated)), nil +} + +func validateDelegationPublicationTempPath(delegationID uuid.UUID, tempPath string) error { + validated, err := validateArtifactRelativePath(tempPath) + if err != nil { + return err + } + prefix := ".tmp-" + delegationID.String() + "-" + if path.Dir(validated) != ".delegations" || + !strings.HasPrefix(path.Base(validated), prefix) { + return ErrArtifactInvalidPath + } + suffix := strings.TrimPrefix(path.Base(validated), prefix) + parsed, err := uuid.Parse(suffix) + if err != nil || parsed == uuid.Nil || parsed.String() != suffix { + return ErrArtifactInvalidPath + } + return nil +} + +func (t *DelegateTool) removeDelegationExchange( + tenantWorkspace string, + delegationID uuid.UUID, +) error { + if tenantWorkspace == "" || delegationID == uuid.Nil { + return ErrArtifactState + } + tenantRoot, err := openArtifactSecureRoot(tenantWorkspace) + if err != nil { + return err + } + defer tenantRoot.close() + relativePath := path.Join("collaboration", "delegations", delegationID.String()) + if err := tenantRoot.removeTree(relativePath); err != nil && !isArtifactNotExist(err) { + return fmt.Errorf("remove delegation exchange: %w", err) + } + return nil +} + +func (t *DelegateTool) tryRemoveDelegationExchange( + tenantWorkspace string, + delegationID uuid.UUID, +) error { + if t.removeExchange != nil { + return t.removeExchange(tenantWorkspace, delegationID) + } + return t.removeDelegationExchange(tenantWorkspace, delegationID) +} diff --git a/internal/tools/delegation_artifact_policy.go b/internal/tools/delegation_artifact_policy.go new file mode 100644 index 00000000..f0c5af73 --- /dev/null +++ b/internal/tools/delegation_artifact_policy.go @@ -0,0 +1,118 @@ +package tools + +import ( + "context" + "fmt" + "path" + "path/filepath" + "strings" +) + +// resolveDelegationInputPath maps the logical inputs/ alias to the captured +// exchange input directory. The host root never needs to be disclosed to the +// delegated model. +func resolveDelegationInputPath(ctx context.Context, raw string) (string, bool, error) { + root := DelegationArtifactInputsFromCtx(ctx) + if root == "" || filepath.IsAbs(raw) { + return "", false, nil + } + logical := path.Clean(filepath.ToSlash(strings.TrimSpace(raw))) + if logical == "inputs" { + return filepath.Clean(root), true, nil + } + if !strings.HasPrefix(logical, "inputs/") { + return "", false, nil + } + relative, err := validateArtifactRelativePath(strings.TrimPrefix(logical, "inputs/")) + if err != nil { + return "", true, err + } + resolved, err := resolvePathWithAllowed( + filepath.FromSlash(relative), + root, + true, + nil, + ) + if err != nil { + return "", true, fmt.Errorf("cannot access delegation input") + } + return resolved, true, nil +} + +func rejectDelegationInputMutation(ctx context.Context, raw string) error { + if _, handled, err := resolveDelegationInputPath(ctx, raw); handled { + if err != nil { + return err + } + return fmt.Errorf("delegation inputs are read-only") + } + return nil +} + +// resolveStructuredMediaPath resolves logical delegation inputs for media +// readers without widening filesystem-tool authority. Outside an artifact run +// it preserves the normal workspace and Agent Team read rules. +func resolveStructuredMediaPath(ctx context.Context, raw, kind string) (string, error) { + return resolveStructuredMediaPathWithAllowed(ctx, raw, kind, allowedWithTeamWorkspace(ctx, nil)) +} + +// resolveStructuredMediaRefPath resolves paths sourced from conversation +// MediaRefs. Unlike explicit user/tool paths, a ref is authoritative only for +// the active workspace or the current delegation's staged inputs. +func resolveStructuredMediaRefPath(ctx context.Context, raw, kind string) (string, error) { + if inputRoot := DelegationArtifactInputsFromCtx(ctx); inputRoot != "" && filepath.IsAbs(raw) { + if resolved, err := resolvePathWithAllowed(raw, inputRoot, true, nil); err == nil { + if err := ValidateRegularFileForRead(resolved); err != nil { + return "", fmt.Errorf("%s delegation input is not a safe regular file", kind) + } + return filepath.Clean(resolved), nil + } + } + return resolveStructuredMediaPathWithAllowed(ctx, raw, kind, nil) +} + +// resolveLoadedMediaRefPath preserves legacy media-store compatibility without +// treating an arbitrary loader return value as filesystem authority. +func resolveLoadedMediaRefPath( + ctx context.Context, + loader MediaPathLoader, + raw, kind string, +) (string, error) { + if rooted, ok := loader.(MediaPathRootProvider); ok { + root := rooted.MediaRootPath() + if root != "" { + if resolved, err := resolvePathWithAllowed(raw, root, true, nil); err == nil { + if err := ValidateRegularFileForRead(resolved); err != nil { + return "", fmt.Errorf("%s legacy media is not a safe regular file", kind) + } + return filepath.Clean(resolved), nil + } + } + } + return resolveStructuredMediaRefPath(ctx, raw, kind) +} + +func resolveStructuredMediaPathWithAllowed(ctx context.Context, raw, kind string, allowedPrefixes []string) (string, error) { + if resolved, handled, err := resolveDelegationInputPath(ctx, raw); handled { + if err != nil { + return "", fmt.Errorf("invalid %s delegation input path", kind) + } + if err := ValidateRegularFileForRead(resolved); err != nil { + return "", fmt.Errorf("%s delegation input is not a safe regular file", kind) + } + return resolved, nil + } + + workspace := ToolWorkspaceFromCtx(ctx) + resolved, err := resolvePathWithAllowed(raw, workspace, effectiveRestrict(ctx, true), allowedPrefixes) + if err != nil { + return "", fmt.Errorf("invalid %s path: %w", kind, err) + } + if err := checkDeniedPath(resolved, workspace, nil); err != nil { + return "", err + } + if err := ValidateRegularFileForRead(resolved); err != nil { + return "", fmt.Errorf("%s path is not a safe regular file: %w", kind, err) + } + return filepath.Clean(resolved), nil +} diff --git a/internal/tools/delegation_artifact_policy_test.go b/internal/tools/delegation_artifact_policy_test.go new file mode 100644 index 00000000..906c1899 --- /dev/null +++ b/internal/tools/delegation_artifact_policy_test.go @@ -0,0 +1,217 @@ +package tools + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/uuid" + + "github.com/nextlevelbuilder/goclaw/internal/bus" + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +func delegationArtifactToolContext(t *testing.T) (context.Context, string, string) { + t.Helper() + root := t.TempDir() + inputs := filepath.Join(root, "inputs") + outputs := filepath.Join(root, "outputs") + if err := os.MkdirAll(inputs, 0750); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(outputs, 0750); err != nil { + t.Fatal(err) + } + ctx := WithDelegationID(context.Background(), uuid.NewString()) + ctx = WithDelegationArtifactInputs(ctx, inputs) + ctx = WithToolWorkspace(ctx, outputs) + return ctx, inputs, outputs +} + +func TestDelegationArtifactFilePolicy(t *testing.T) { + ctx, inputs, outputs := delegationArtifactToolContext(t) + if err := os.WriteFile(filepath.Join(inputs, "source.txt"), []byte("staged"), 0440); err != nil { + t.Fatal(err) + } + + read := NewReadFileTool(outputs, true).Execute(ctx, map[string]any{"path": "inputs/source.txt"}) + if read.IsError || !strings.Contains(read.ForLLM, "staged") { + t.Fatalf("read staged input = %#v", read) + } + listed := NewListFilesTool(outputs, true).Execute(ctx, map[string]any{"path": "inputs"}) + if listed.IsError || !strings.Contains(listed.ForLLM, "source.txt") { + t.Fatalf("list staged inputs = %#v", listed) + } + + writeInput := NewWriteFileTool(outputs, true).Execute(ctx, map[string]any{ + "path": "inputs/source.txt", "content": "mutated", + }) + if !writeInput.IsError || !strings.Contains(writeInput.ForLLM, "read-only") { + t.Fatalf("input mutation = %#v, want read-only error", writeInput) + } + editInput := NewEditTool(outputs, true).Execute(ctx, map[string]any{ + "path": "inputs/source.txt", "old_string": "staged", "new_string": "mutated", + }) + if !editInput.IsError || !strings.Contains(editInput.ForLLM, "read-only") { + t.Fatalf("input edit = %#v, want read-only error", editInput) + } + + written := NewWriteFileTool(outputs, true).Execute(ctx, map[string]any{ + "path": "report.txt", "content": "result", "deliver": true, + }) + if written.IsError || len(written.Media) != 0 { + t.Fatalf("prepublication write delivery = %#v", written) + } + if got, err := os.ReadFile(filepath.Join(outputs, "report.txt")); err != nil || string(got) != "result" { + t.Fatalf("output file = %q, %v", got, err) + } + + if result := NewSendFileTool(outputs, true).Execute(ctx, map[string]any{"path": "report.txt"}); !result.IsError { + t.Fatalf("send_file published prepublication output: %#v", result) + } + if result := (&MessageTool{}).Execute(ctx, map[string]any{ + "action": "send", "message": "MEDIA:report.txt", + }); !result.IsError { + t.Fatalf("message published prepublication output: %#v", result) + } +} + +func TestReadDocumentArchiveUsesLogicalDelegationInputPath(t *testing.T) { + ctx, inputs, outputs := delegationArtifactToolContext(t) + archive := filepath.Join(inputs, "archive.zip") + if err := os.WriteFile(archive, []byte("not a real zip"), 0440); err != nil { + t.Fatal(err) + } + + result := NewReadDocumentTool(nil, nil).Execute(ctx, map[string]any{ + "path": "inputs/archive.zip", + "prompt": "inspect archive", + }) + if result.IsError { + t.Fatalf("read_document archive = %#v", result) + } + if strings.Contains(result.ForLLM, inputs) || strings.Contains(result.ForLLM, outputs) { + t.Fatalf("archive result leaked host path: %s", result.ForLLM) + } + if !strings.Contains(result.ForLLM, "inputs/archive.zip") { + t.Fatalf("archive result omitted logical path: %s", result.ForLLM) + } +} + +func TestDelegationArtifactDisablesVirtualMemoryRouting(t *testing.T) { + ctx, _, outputs := delegationArtifactToolContext(t) + agentID := uuid.New() + ctx = store.WithAgentID(ctx, agentID) + ctx = store.WithUserID(ctx, "user-1") + memStore := newMockMemoryStore() + memStore.docs[docKey(agentID.String(), "user-1", "MEMORY.md")] = "database-secret" + interceptor := NewMemoryInterceptor(memStore, outputs) + + if err := os.WriteFile(filepath.Join(outputs, "MEMORY.md"), []byte("physical-output"), 0640); err != nil { + t.Fatal(err) + } + reader := NewReadFileTool(outputs, true) + reader.SetMemoryInterceptor(interceptor) + result := reader.Execute(ctx, map[string]any{"path": "MEMORY.md"}) + if result.IsError || strings.Contains(result.ForLLM, "database-secret") || + !strings.Contains(result.ForLLM, "physical-output") { + t.Fatalf("delegation memory read routed virtually: %#v", result) + } + + writer := NewWriteFileTool(outputs, true) + writer.SetMemoryInterceptor(interceptor) + result = writer.Execute(ctx, map[string]any{"path": "MEMORY.md", "content": "new-output"}) + if result.IsError { + t.Fatalf("physical MEMORY.md write failed: %#v", result) + } + if got := memStore.docs[docKey(agentID.String(), "user-1", "MEMORY.md")]; got != "database-secret" { + t.Fatalf("virtual memory mutated: %q", got) + } +} + +func TestDelegationStructuredMediaPathsResolveOnlyStagedInputs(t *testing.T) { + ctx, inputs, _ := delegationArtifactToolContext(t) + staged := filepath.Join(inputs, "clip.mp4") + if err := os.WriteFile(staged, []byte("video"), 0440); err != nil { + t.Fatal(err) + } + + got, err := resolveStructuredMediaPath(ctx, "inputs/clip.mp4", "video") + if err != nil { + t.Fatalf("resolve staged media: %v", err) + } + want, err := filepath.EvalSymlinks(staged) + if err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("resolved path = %q, want %q", got, want) + } + + _, err = resolveStructuredMediaPath(ctx, "inputs/missing.mp4", "video") + if err == nil { + t.Fatal("missing staged media unexpectedly resolved") + } + if strings.Contains(err.Error(), filepath.Dir(inputs)) { + t.Fatalf("error leaked exchange host root: %v", err) + } +} + +func TestCreateImageReferenceUsesLogicalDelegationInputPath(t *testing.T) { + ctx, inputs, _ := delegationArtifactToolContext(t) + staged := filepath.Join(inputs, "reference.png") + want := []byte("staged-reference") + if err := os.WriteFile(staged, want, 0440); err != nil { + t.Fatal(err) + } + + refs, err := NewCreateImageTool(nil).resolveReferenceImages(ctx, map[string]any{ + "ref_images": []any{map[string]any{"path": "inputs/reference.png"}}, + }) + if err != nil { + t.Fatalf("resolve staged create_image reference: %v", err) + } + if len(refs) != 1 || string(refs[0].Data) != string(want) { + t.Fatalf("resolved refs = %#v, want staged reference", refs) + } +} + +func TestDelegationArtifactResultPolicySuppressesUnpublishedMedia(t *testing.T) { + ctx, _, outputs := delegationArtifactToolContext(t) + result := &Result{ + ForLLM: "created\nMEDIA:" + filepath.Join(outputs, "generated.png") + "\nkeep this", + Media: []bus.MediaFile{{ + Path: filepath.Join(outputs, "generated.png"), + }}, + } + + ApplyDelegationArtifactResultPolicy(ctx, result) + + if len(result.Media) != 0 { + t.Fatalf("unpublished media remained: %#v", result.Media) + } + if strings.Contains(result.ForLLM, "MEDIA:") || !strings.Contains(result.ForLLM, "keep this") { + t.Fatalf("artifact result policy = %q", result.ForLLM) + } +} + +func TestDelegationArtifactResultPolicyPreservesMediaDiscussion(t *testing.T) { + ctx := delegationArtifactTestContext() + result := &Result{ + ForLLM: "The literal marker MEDIA: is documented here.\nGenerated MEDIA:outputs/report.pdf successfully.", + } + + ApplyDelegationArtifactResultPolicy(ctx, result) + + if !strings.Contains(result.ForLLM, "The literal marker MEDIA: is documented here.") { + t.Fatalf("legitimate MEDIA discussion was removed: %q", result.ForLLM) + } + if strings.Contains(result.ForLLM, "MEDIA:outputs/report.pdf") { + t.Fatalf("unpublished media path leaked: %q", result.ForLLM) + } + if !strings.Contains(result.ForLLM, "Generated successfully.") { + t.Fatalf("surrounding tool result text was removed: %q", result.ForLLM) + } +} diff --git a/internal/tools/delegation_artifact_result_policy.go b/internal/tools/delegation_artifact_result_policy.go new file mode 100644 index 00000000..3004b054 --- /dev/null +++ b/internal/tools/delegation_artifact_result_policy.go @@ -0,0 +1,36 @@ +package tools + +import ( + "context" + "strings" +) + +// ApplyDelegationArtifactResultPolicy keeps unvalidated delegate outputs inside +// the exchange. Publication is owned by DelegateTool after the child run +// succeeds and the manifest has been validated. +func ApplyDelegationArtifactResultPolicy(ctx context.Context, result *Result) { + if result == nil || !IsDelegationArtifactRun(ctx) { + return + } + result.Media = nil + result.ForLLM = stripArtifactMediaLines(result.ForLLM) +} + +func stripArtifactMediaLines(content string) string { + if !strings.Contains(content, "MEDIA:") { + return content + } + lines := strings.Split(content, "\n") + kept := make([]string, 0, len(lines)) + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "[[audio_as_voice]]") { + continue + } + cleaned := strings.TrimRight(embeddedMediaPattern.ReplaceAllString(line, ""), " \t") + if strings.TrimSpace(cleaned) != "" { + kept = append(kept, cleaned) + } + } + return strings.TrimSpace(strings.Join(kept, "\n")) +} diff --git a/internal/tools/delegation_artifacts.go b/internal/tools/delegation_artifacts.go new file mode 100644 index 00000000..27baca0b --- /dev/null +++ b/internal/tools/delegation_artifacts.go @@ -0,0 +1,323 @@ +package tools + +import ( + "context" + "errors" + "fmt" + "path" + "strings" + "sync" + "time" + "unicode" + + "github.com/google/uuid" +) + +const ( + DelegationArtifactManifestVersion = 1 + DelegationArtifactMaxFileBytes = int64(50 * 1024 * 1024) + DelegationArtifactMaxTotalBytes = int64(100 * 1024 * 1024) + DelegationArtifactMaxFiles = 100 +) + +var ( + ErrArtifactInvalidPath = errors.New("invalid delegation artifact path") + ErrArtifactNonRegular = errors.New("delegation artifact is not a regular file") + ErrArtifactHardlink = errors.New("delegation artifact has multiple hard links") + ErrArtifactLimitExceeded = errors.New("delegation artifact limit exceeded") + ErrArtifactPublishConflict = errors.New("delegation artifact publication conflict") + ErrArtifactSecureUnavailable = errors.New("secure delegation artifact filesystem unavailable") + ErrArtifactState = errors.New("invalid delegation artifact state") +) + +// DelegationArtifactError is safe to return across runtime boundaries: Path is +// always a logical relative path and never a host workspace path. +type DelegationArtifactError struct { + Code string + Op string + Path string + Err error +} + +func (e *DelegationArtifactError) Error() string { + switch { + case e.Path != "": + return fmt.Sprintf("%s %q: %v", e.Op, e.Path, e.Err) + case e.Op != "": + return fmt.Sprintf("%s: %v", e.Op, e.Err) + default: + return e.Err.Error() + } +} + +func (e *DelegationArtifactError) Unwrap() error { return e.Err } + +type DelegationArtifactLimits struct { + MaxFileBytes int64 + MaxTotalBytes int64 + MaxFiles int +} + +func DefaultDelegationArtifactLimits() DelegationArtifactLimits { + return DelegationArtifactLimits{ + MaxFileBytes: DelegationArtifactMaxFileBytes, + MaxTotalBytes: DelegationArtifactMaxTotalBytes, + MaxFiles: DelegationArtifactMaxFiles, + } +} + +func (l DelegationArtifactLimits) validate() (DelegationArtifactLimits, error) { + if l == (DelegationArtifactLimits{}) { + return DefaultDelegationArtifactLimits(), nil + } + if l.MaxFileBytes <= 0 || l.MaxTotalBytes <= 0 || l.MaxFiles <= 0 { + return DelegationArtifactLimits{}, artifactError( + "artifact_invalid_limits", "configure", "", ErrArtifactLimitExceeded, + ) + } + if l.MaxFileBytes > l.MaxTotalBytes { + return DelegationArtifactLimits{}, artifactError( + "artifact_invalid_limits", "configure", "", ErrArtifactLimitExceeded, + ) + } + return l, nil +} + +type DelegationArtifactStatus string + +const ( + DelegationArtifactStaged DelegationArtifactStatus = "staged" + DelegationArtifactPublished DelegationArtifactStatus = "published" +) + +type DelegationArtifact struct { + Path string `json:"path"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + MediaType string `json:"media_type"` + Status DelegationArtifactStatus `json:"status"` +} + +type DelegationArtifactManifestOutput struct { + Path string `json:"path"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + MediaType string `json:"media_type"` +} + +// DelegationArtifactManifest is the durable, workspace-safe manifest. It +// deliberately excludes tenant IDs, agent IDs, host paths, prompts, and traces. +type DelegationArtifactManifest struct { + SchemaVersion int `json:"schema_version"` + DelegationID string `json:"delegation_id"` + PublishedAt time.Time `json:"published_at"` + OutputCount int `json:"output_count"` + OutputBytes int64 `json:"output_bytes"` + Outputs []DelegationArtifactManifestOutput `json:"outputs"` +} + +type DelegationArtifactPublication struct { + RootPath string + ManifestPath string + Manifest DelegationArtifactManifest +} + +type DelegationArtifactMount struct { + LogicalAlias string + HostRoot string + ContainerPath string + ReadOnly bool +} + +type DelegationArtifactFailureRetention struct { + FailedAt time.Time + RetainUntil time.Time + ReasonCode string + PublicationTempPath string +} + +type delegationArtifactExchangeState string + +const ( + artifactExchangeOpen delegationArtifactExchangeState = "open" + artifactExchangeFailed delegationArtifactExchangeState = "failed_retained" + artifactExchangePublished delegationArtifactExchangeState = "published" +) + +// DelegationArtifactRoot retains a secure directory handle so later renames or +// replacements of the path used to open it cannot redirect artifact access. +type DelegationArtifactRoot struct { + root *artifactSecureRoot +} + +func OpenDelegationArtifactRoot(hostPath string) (*DelegationArtifactRoot, error) { + root, err := openArtifactSecureRoot(hostPath) + if err != nil { + return nil, wrapArtifactFilesystemError("capture_root", "", err) + } + return &DelegationArtifactRoot{root: root}, nil +} + +func (r *DelegationArtifactRoot) Close() error { + if r == nil || r.root == nil { + return nil + } + return r.root.close() +} + +// DelegationArtifactExchange owns one tenant-scoped, delegation-scoped +// inputs/outputs exchange. Host paths are runtime-only and never serialized. +type DelegationArtifactExchange struct { + tenantID uuid.UUID + delegationID uuid.UUID + hostRoot string + root *artifactSecureRoot + limits DelegationArtifactLimits + failureTTL time.Duration + + mu sync.Mutex + state delegationArtifactExchangeState + inputCount int + inputBytes int64 + inputs []DelegationArtifact + failure *DelegationArtifactFailureRetention + publicationTempPath string +} + +func (e *DelegationArtifactExchange) DelegationID() uuid.UUID { return e.delegationID } +func (e *DelegationArtifactExchange) TenantID() uuid.UUID { return e.tenantID } + +// InputsMount is runtime wiring for the delegated sandbox. Do not include its +// HostRoot in prompts, tool results, logs, traces, or durable manifests. +func (e *DelegationArtifactExchange) InputsMount() DelegationArtifactMount { + return DelegationArtifactMount{ + LogicalAlias: "inputs", + HostRoot: e.inputsHostPath(), + ContainerPath: "/workspace/inputs", + ReadOnly: true, + } +} + +// OutputsHostPath is the delegated run's ephemeral read/write workspace. +func (e *DelegationArtifactExchange) OutputsHostPath() string { + return e.outputsHostPath() +} + +func (e *DelegationArtifactExchange) Close() error { + if e == nil || e.root == nil { + return nil + } + return e.root.close() +} + +func (e *DelegationArtifactExchange) FailureRetention() (DelegationArtifactFailureRetention, bool) { + e.mu.Lock() + defer e.mu.Unlock() + if e.failure == nil { + return DelegationArtifactFailureRetention{}, false + } + return *e.failure, true +} + +func (e *DelegationArtifactExchange) ReadyForCleanup(now time.Time) bool { + retention, ok := e.FailureRetention() + return ok && !now.Before(retention.RetainUntil) +} + +func (e *DelegationArtifactExchange) RetainFailure(now time.Time, reasonCode string) { + e.mu.Lock() + defer e.mu.Unlock() + e.retainFailureLocked(now, reasonCode) +} + +func (e *DelegationArtifactExchange) retainFailureLocked(now time.Time, reasonCode string) { + if e.state == artifactExchangePublished { + return + } + if e.failure != nil { + return + } + if reasonCode == "" { + reasonCode = "artifact_exchange_failed" + } + e.state = artifactExchangeFailed + e.failure = &DelegationArtifactFailureRetention{ + FailedAt: now.UTC(), + RetainUntil: now.UTC().Add(e.failureTTL), + ReasonCode: reasonCode, + PublicationTempPath: e.publicationTempPath, + } +} + +func artifactError(code, op, logicalPath string, err error) error { + return &DelegationArtifactError{ + Code: code, + Op: op, + Path: logicalPath, + Err: err, + } +} + +func wrapArtifactFilesystemError(op, logicalPath string, err error) error { + if err == nil { + return nil + } + if errors.Is(err, ErrArtifactSecureUnavailable) { + return artifactError("artifact_secure_open_unavailable", op, logicalPath, err) + } + if errors.Is(err, ErrArtifactPublishConflict) { + return artifactError("artifact_publish_conflict", op, logicalPath, err) + } + return artifactError("artifact_filesystem_error", op, logicalPath, err) +} + +func validateArtifactRelativePath(raw string) (string, error) { + if raw == "" || + strings.Contains(raw, `\`) || + strings.Contains(raw, ":") || + strings.IndexFunc(raw, unicode.IsControl) >= 0 { + return "", artifactError("artifact_invalid_path", "validate_path", raw, ErrArtifactInvalidPath) + } + if strings.HasPrefix(raw, "/") { + return "", artifactError("artifact_invalid_path", "validate_path", raw, ErrArtifactInvalidPath) + } + clean := path.Clean(raw) + if clean == "." || clean == ".." || strings.HasPrefix(clean, "../") || clean != raw { + return "", artifactError("artifact_invalid_path", "validate_path", raw, ErrArtifactInvalidPath) + } + for component := range strings.SplitSeq(clean, "/") { + if component == "" || + component == "." || + component == ".." || + strings.HasSuffix(component, ".") || + strings.HasSuffix(component, " ") || + isReservedArtifactComponent(component) { + return "", artifactError("artifact_invalid_path", "validate_path", raw, ErrArtifactInvalidPath) + } + } + return clean, nil +} + +func isReservedArtifactComponent(component string) bool { + base := component + if dot := strings.IndexByte(base, '.'); dot >= 0 { + base = base[:dot] + } + switch strings.ToUpper(base) { + case "CON", "PRN", "AUX", "NUL", "CLOCK$", + "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", + "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9": + return true + default: + return false + } +} + +func checkArtifactContext(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + return nil + } +} diff --git a/internal/tools/delegation_artifacts_copy.go b/internal/tools/delegation_artifacts_copy.go new file mode 100644 index 00000000..b1399bc8 --- /dev/null +++ b/internal/tools/delegation_artifacts_copy.go @@ -0,0 +1,136 @@ +package tools + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "net/http" + "os" +) + +type artifactSniffWriter struct { + buf []byte +} + +func (w *artifactSniffWriter) Write(p []byte) (int, error) { + if len(w.buf) < 512 { + remaining := min(512-len(w.buf), len(p)) + w.buf = append(w.buf, p[:remaining]...) + } + return len(p), nil +} + +func copyArtifactFile( + ctx context.Context, + sourceRoot *artifactSecureRoot, + sourcePath string, + destinationRoot *artifactSecureRoot, + destinationPath string, + maxFileBytes int64, + maxRemainingBytes int64, + destinationMode os.FileMode, +) (DelegationArtifact, error) { + if maxRemainingBytes < 0 { + return DelegationArtifact{}, artifactError( + "artifact_total_byte_limit", "copy", destinationPath, ErrArtifactLimitExceeded, + ) + } + source, err := sourceRoot.openEntry(sourcePath) + if err != nil { + return DelegationArtifact{}, classifyArtifactOpenError("open_source", sourcePath, err) + } + defer source.close() + if source.kind != artifactEntryRegular { + return DelegationArtifact{}, artifactError( + "artifact_non_regular", "open_source", sourcePath, ErrArtifactNonRegular, + ) + } + if source.links != 1 { + return DelegationArtifact{}, artifactError( + "artifact_hardlink", "open_source", sourcePath, ErrArtifactHardlink, + ) + } + effectiveLimit := min(maxRemainingBytes, maxFileBytes) + if source.size > effectiveLimit { + return DelegationArtifact{}, artifactError( + "artifact_byte_limit", "copy", destinationPath, ErrArtifactLimitExceeded, + ) + } + + destination, err := destinationRoot.createFile(destinationPath, 0600) + if err != nil { + return DelegationArtifact{}, wrapArtifactFilesystemError("create_destination", destinationPath, err) + } + closeDestination := true + defer func() { + if closeDestination { + _ = destination.Close() + } + }() + + hash := sha256.New() + sniff := &artifactSniffWriter{} + writer := io.MultiWriter(destination, hash, sniff) + reader := io.LimitReader(&contextArtifactReader{ctx: ctx, reader: source.file}, effectiveLimit+1) + written, err := io.Copy(writer, reader) + if err != nil { + return DelegationArtifact{}, artifactError("artifact_copy_failed", "copy", destinationPath, err) + } + if written > effectiveLimit { + return DelegationArtifact{}, artifactError( + "artifact_byte_limit", "copy", destinationPath, ErrArtifactLimitExceeded, + ) + } + if err := destination.Sync(); err != nil { + return DelegationArtifact{}, artifactError( + "artifact_sync_failed", "sync_file", destinationPath, err, + ) + } + if err := destination.Chmod(destinationMode); err != nil { + return DelegationArtifact{}, artifactError( + "artifact_chmod_failed", "protect_file", destinationPath, err, + ) + } + if err := destination.Close(); err != nil { + return DelegationArtifact{}, artifactError( + "artifact_close_failed", "close_file", destinationPath, err, + ) + } + closeDestination = false + + mediaType := "application/octet-stream" + if len(sniff.buf) > 0 { + mediaType = http.DetectContentType(sniff.buf) + } + return DelegationArtifact{ + Path: destinationPath, + SizeBytes: written, + SHA256: hex.EncodeToString(hash.Sum(nil)), + MediaType: mediaType, + }, nil +} + +type contextArtifactReader struct { + ctx context.Context + reader io.Reader +} + +func (r *contextArtifactReader) Read(p []byte) (int, error) { + if err := checkArtifactContext(r.ctx); err != nil { + return 0, err + } + return r.reader.Read(p) +} + +func classifyArtifactOpenError(op, logicalPath string, err error) error { + switch { + case errors.Is(err, errArtifactSymlink): + return artifactError("artifact_symlink", op, logicalPath, ErrArtifactNonRegular) + case errors.Is(err, errArtifactReparsePoint): + return artifactError("artifact_reparse_point", op, logicalPath, ErrArtifactNonRegular) + default: + return wrapArtifactFilesystemError(op, logicalPath, err) + } +} diff --git a/internal/tools/delegation_artifacts_darwin.go b/internal/tools/delegation_artifacts_darwin.go new file mode 100644 index 00000000..7bc6fbf6 --- /dev/null +++ b/internal/tools/delegation_artifacts_darwin.go @@ -0,0 +1,32 @@ +//go:build darwin + +package tools + +import ( + "errors" + + "golang.org/x/sys/unix" +) + +func (r *artifactSecureRoot) renameNoReplace(oldName, newName string) error { + oldPath, err := validateArtifactRelativePath(oldName) + if err != nil { + return err + } + newPath, err := validateArtifactRelativePath(newName) + if err != nil { + return err + } + err = unix.RenameatxNp(r.fd, oldPath, r.fd, newPath, unix.RENAME_EXCL) + if errors.Is(err, unix.EEXIST) { + return errors.Join(ErrArtifactPublishConflict, err) + } + if errors.Is(err, unix.ENOSYS) || errors.Is(err, unix.EINVAL) { + return errors.Join(ErrArtifactSecureUnavailable, err) + } + return err +} + +func isArtifactNotExist(err error) bool { + return errors.Is(err, unix.ENOENT) +} diff --git a/internal/tools/delegation_artifacts_exchange.go b/internal/tools/delegation_artifacts_exchange.go new file mode 100644 index 00000000..e97b2017 --- /dev/null +++ b/internal/tools/delegation_artifacts_exchange.go @@ -0,0 +1,193 @@ +package tools + +import ( + "context" + "errors" + "fmt" + "path" + "path/filepath" + "strings" + "time" + + "github.com/google/uuid" +) + +const defaultDelegationArtifactFailureTTL = 60 * time.Minute + +// NewDelegationArtifactExchange creates: +// +// /collaboration/delegations//{inputs,outputs} +// +// tenantWorkspace must already be resolved through the canonical tenant path +// helper. Both IDs are mandatory non-zero UUIDs. +func NewDelegationArtifactExchange( + tenantWorkspace string, + tenantID uuid.UUID, + delegationID uuid.UUID, + limits DelegationArtifactLimits, + failureTTL time.Duration, +) (*DelegationArtifactExchange, error) { + if tenantWorkspace == "" || tenantID == uuid.Nil || delegationID == uuid.Nil { + return nil, artifactError( + "artifact_invalid_identity", "create_exchange", "", ErrArtifactInvalidPath, + ) + } + validLimits, err := limits.validate() + if err != nil { + return nil, err + } + if failureTTL <= 0 { + failureTTL = defaultDelegationArtifactFailureTTL + } + + tenantRoot, err := openArtifactSecureRoot(tenantWorkspace) + if err != nil { + return nil, wrapArtifactFilesystemError("capture_tenant_root", "", err) + } + defer tenantRoot.close() + + const exchangeParent = "collaboration/delegations" + if err := tenantRoot.mkdirAll(exchangeParent, 0750); err != nil { + return nil, wrapArtifactFilesystemError("create_exchange_parent", "", err) + } + exchangeRelative := path.Join(exchangeParent, delegationID.String()) + exchangeRoot, err := tenantRoot.createSubroot(exchangeRelative, 0700) + if err != nil { + if isArtifactAlreadyExists(err) { + return nil, artifactError( + "artifact_exchange_conflict", "create_exchange", "", ErrArtifactPublishConflict, + ) + } + return nil, wrapArtifactFilesystemError("create_exchange", "", err) + } + if err := exchangeRoot.mkdirAll("inputs", 0750); err != nil { + exchangeRoot.close() + return nil, wrapArtifactFilesystemError("create_inputs", "", err) + } + if err := exchangeRoot.mkdirAll("outputs", 0700); err != nil { + exchangeRoot.close() + return nil, wrapArtifactFilesystemError("create_outputs", "", err) + } + + exchange := &DelegationArtifactExchange{ + tenantID: tenantID, + delegationID: delegationID, + hostRoot: filepath.Join(tenantWorkspace, filepath.FromSlash(exchangeRelative)), + root: exchangeRoot, + limits: validLimits, + failureTTL: failureTTL, + state: artifactExchangeOpen, + } + startedAt := time.Now().UTC() + lifecycleState := newStagingDelegationArtifactLifecycleState( + tenantID, + delegationID, + startedAt, + startedAt.Add(failureTTL), + ) + if err := exchange.persistLifecycleState(lifecycleState); err != nil { + _ = exchange.Close() + _ = tenantRoot.removeTree(exchangeRelative) + return nil, err + } + return exchange, nil +} + +func (e *DelegationArtifactExchange) inputsHostPath() string { + return filepath.Join(e.hostRoot, "inputs") +} + +func (e *DelegationArtifactExchange) outputsHostPath() string { + return filepath.Join(e.hostRoot, "outputs") +} + +// StageInputs copies already-authorized relative files from the captured caller +// workspace. Duplicate basenames receive deterministic suffixes. +func (e *DelegationArtifactExchange) StageInputs( + ctx context.Context, + source *DelegationArtifactRoot, + relativePaths []string, +) ([]DelegationArtifact, error) { + if source == nil || source.root == nil { + return nil, artifactError("artifact_missing_source_root", "stage_inputs", "", ErrArtifactState) + } + e.mu.Lock() + defer e.mu.Unlock() + if e.state != artifactExchangeOpen { + return nil, artifactError("artifact_invalid_state", "stage_inputs", "", ErrArtifactState) + } + if len(relativePaths) > e.limits.MaxFiles-e.inputCount { + err := artifactError("artifact_file_limit", "stage_inputs", "", ErrArtifactLimitExceeded) + e.retainFailureLocked(time.Now(), artifactErrorCode(err)) + return nil, err + } + + usedNames := make(map[string]struct{}, len(e.inputs)+len(relativePaths)) + for _, input := range e.inputs { + usedNames[strings.TrimPrefix(input.Path, "inputs/")] = struct{}{} + } + staged := make([]DelegationArtifact, 0, len(relativePaths)) + for _, rawPath := range relativePaths { + if err := checkArtifactContext(ctx); err != nil { + e.retainFailureLocked(time.Now(), "artifact_context_cancelled") + return nil, err + } + sourcePath, err := validateArtifactRelativePath(rawPath) + if err != nil { + e.retainFailureLocked(time.Now(), artifactErrorCode(err)) + return nil, err + } + name := nextArtifactInputName(path.Base(sourcePath), usedNames) + logicalPath := path.Join("inputs", name) + remaining := e.limits.MaxTotalBytes - e.inputBytes + artifact, err := copyArtifactFile( + ctx, + source.root, + sourcePath, + e.root, + logicalPath, + e.limits.MaxFileBytes, + remaining, + 0440, + ) + if err != nil { + e.retainFailureLocked(time.Now(), artifactErrorCode(err)) + return nil, err + } + artifact.Status = DelegationArtifactStaged + e.inputCount++ + e.inputBytes += artifact.SizeBytes + e.inputs = append(e.inputs, artifact) + staged = append(staged, artifact) + usedNames[name] = struct{}{} + } + return append([]DelegationArtifact(nil), staged...), nil +} + +func (e *DelegationArtifactExchange) StagedInputs() []DelegationArtifact { + e.mu.Lock() + defer e.mu.Unlock() + return append([]DelegationArtifact(nil), e.inputs...) +} + +func nextArtifactInputName(base string, used map[string]struct{}) string { + if _, exists := used[base]; !exists { + return base + } + ext := path.Ext(base) + stem := strings.TrimSuffix(base, ext) + for i := 2; ; i++ { + candidate := fmt.Sprintf("%s-%d%s", stem, i, ext) + if _, exists := used[candidate]; !exists { + return candidate + } + } +} + +func artifactErrorCode(err error) string { + var artifactErr *DelegationArtifactError + if errors.As(err, &artifactErr) && artifactErr.Code != "" { + return artifactErr.Code + } + return "artifact_exchange_failed" +} diff --git a/internal/tools/delegation_artifacts_hostile_posix_test.go b/internal/tools/delegation_artifacts_hostile_posix_test.go new file mode 100644 index 00000000..790cadb5 --- /dev/null +++ b/internal/tools/delegation_artifacts_hostile_posix_test.go @@ -0,0 +1,37 @@ +//go:build linux || darwin + +package tools + +import ( + "context" + "errors" + "path/filepath" + "testing" + "time" + + "github.com/google/uuid" + "golang.org/x/sys/unix" +) + +func TestDelegationArtifactPublishRejectsFIFOWithoutBlocking(t *testing.T) { + exchange := newTestDelegationExchange( + t, + t.TempDir(), + uuid.New(), + uuid.New(), + DelegationArtifactLimits{}, + ) + fifoPath := filepath.Join(exchange.OutputsHostPath(), "pipe") + if err := unix.Mkfifo(fifoPath, 0600); err != nil { + t.Skipf("mkfifo unavailable: %v", err) + } + destinationPath := t.TempDir() + _, err := exchange.Publish( + context.Background(), + openTestArtifactRoot(t, destinationPath), + time.Now(), + ) + if !errors.Is(err, ErrArtifactNonRegular) { + t.Fatalf("Publish() error = %v, want non-regular rejection", err) + } +} diff --git a/internal/tools/delegation_artifacts_lifecycle.go b/internal/tools/delegation_artifacts_lifecycle.go new file mode 100644 index 00000000..d2d4eb24 --- /dev/null +++ b/internal/tools/delegation_artifacts_lifecycle.go @@ -0,0 +1,195 @@ +package tools + +import ( + "encoding/json" + "fmt" + "io" + "time" + + "github.com/google/uuid" +) + +const ( + delegationArtifactLifecycleVersion = 2 + delegationArtifactLifecycleFile = ".goclaw-lifecycle.json" + delegationArtifactLifecycleMaxBytes = 64 * 1024 +) + +type delegationArtifactLifecycleStatus string + +const ( + artifactLifecycleStaging delegationArtifactLifecycleStatus = "staging" + artifactLifecycleRunning delegationArtifactLifecycleStatus = "running" + artifactLifecyclePublishing delegationArtifactLifecycleStatus = "publishing" + artifactLifecyclePublished delegationArtifactLifecycleStatus = "published" + artifactLifecycleFailed delegationArtifactLifecycleStatus = "failed" + artifactLifecycleCancelled delegationArtifactLifecycleStatus = "cancelled" +) + +type delegationArtifactCallerLocation struct { + Base string `json:"base"` + RelativePath string `json:"relative_path"` +} + +// delegationArtifactLifecycleState contains only logical locations and stable +// identifiers. Host roots are reconstructed from runtime configuration and are +// never serialized. +type delegationArtifactLifecycleState struct { + SchemaVersion int `json:"schema_version"` + DelegationID string `json:"delegation_id"` + TenantID string `json:"tenant_id"` + TenantSlug string `json:"tenant_slug,omitempty"` + Status delegationArtifactLifecycleStatus `json:"status"` + StartedAt time.Time `json:"started_at"` + FailedAt *time.Time `json:"failed_at,omitempty"` + RetainUntil time.Time `json:"retain_until"` + ReasonCode string `json:"reason_code"` + CallerLocation *delegationArtifactCallerLocation `json:"caller_location,omitempty"` + PublicationTempPath string `json:"publication_temp_path,omitempty"` +} + +func newStagingDelegationArtifactLifecycleState( + tenantID uuid.UUID, + delegationID uuid.UUID, + startedAt time.Time, + retainUntil time.Time, +) delegationArtifactLifecycleState { + return delegationArtifactLifecycleState{ + SchemaVersion: delegationArtifactLifecycleVersion, + DelegationID: delegationID.String(), + TenantID: tenantID.String(), + Status: artifactLifecycleStaging, + StartedAt: startedAt.UTC(), + RetainUntil: retainUntil.UTC(), + ReasonCode: "artifact_staging", + } +} + +func persistDelegationArtifactLifecycleState( + root *artifactSecureRoot, + delegationID uuid.UUID, + state delegationArtifactLifecycleState, +) error { + if root == nil { + return artifactError("artifact_lifecycle_state_failed", "write_lifecycle_state", "", ErrArtifactState) + } + if err := validateDelegationArtifactLifecycleState(state, delegationID); err != nil { + return err + } + encoded, err := json.Marshal(state) + if err != nil { + return artifactError("artifact_lifecycle_state_failed", "marshal_lifecycle_state", "", err) + } + encoded = append(encoded, '\n') + tempName := ".goclaw-lifecycle-" + uuid.NewString() + ".tmp" + stateFile, err := root.createFile(tempName, 0600) + if err != nil { + return wrapArtifactFilesystemError("create_lifecycle_state", "", err) + } + removeTemp := true + defer func() { + _ = stateFile.Close() + if removeTemp { + _ = root.removeTree(tempName) + } + }() + if _, err := stateFile.Write(encoded); err != nil { + return artifactError("artifact_lifecycle_state_failed", "write_lifecycle_state", "", err) + } + if err := stateFile.Sync(); err != nil { + return artifactError("artifact_lifecycle_state_failed", "sync_lifecycle_state", "", err) + } + if err := stateFile.Close(); err != nil { + return artifactError("artifact_lifecycle_state_failed", "close_lifecycle_state", "", err) + } + if err := root.renameReplace(tempName, delegationArtifactLifecycleFile); err != nil { + return wrapArtifactFilesystemError("publish_lifecycle_state", "", err) + } + removeTemp = false + if err := root.syncDir("."); err != nil { + return artifactError("artifact_lifecycle_state_failed", "sync_lifecycle_directory", "", err) + } + return nil +} + +func (e *DelegationArtifactExchange) persistLifecycleState(state delegationArtifactLifecycleState) error { + if e == nil { + return artifactError("artifact_lifecycle_state_failed", "write_lifecycle_state", "", ErrArtifactState) + } + return persistDelegationArtifactLifecycleState(e.root, e.delegationID, state) +} + +func readDelegationArtifactLifecycleState( + root *artifactSecureRoot, + delegationID uuid.UUID, +) (delegationArtifactLifecycleState, error) { + entry, err := root.openEntry(delegationArtifactLifecycleFile) + if err != nil { + return delegationArtifactLifecycleState{}, err + } + defer entry.close() + if entry.kind != artifactEntryRegular || entry.links != 1 || + entry.size <= 0 || entry.size > delegationArtifactLifecycleMaxBytes { + return delegationArtifactLifecycleState{}, ErrArtifactState + } + encoded, err := io.ReadAll(io.LimitReader(entry.file, delegationArtifactLifecycleMaxBytes+1)) + if err != nil { + return delegationArtifactLifecycleState{}, err + } + if len(encoded) > delegationArtifactLifecycleMaxBytes { + return delegationArtifactLifecycleState{}, ErrArtifactState + } + var state delegationArtifactLifecycleState + if err := json.Unmarshal(encoded, &state); err != nil { + return delegationArtifactLifecycleState{}, err + } + if err := validateDelegationArtifactLifecycleState(state, delegationID); err != nil { + return delegationArtifactLifecycleState{}, err + } + return state, nil +} + +func validateDelegationArtifactLifecycleState( + state delegationArtifactLifecycleState, + delegationID uuid.UUID, +) error { + if state.SchemaVersion != delegationArtifactLifecycleVersion || + state.DelegationID != delegationID.String() || + state.TenantID == "" || + state.StartedAt.IsZero() || + state.RetainUntil.IsZero() || + state.ReasonCode == "" { + return artifactError("artifact_lifecycle_state_invalid", "validate_lifecycle_state", "", ErrArtifactState) + } + switch state.Status { + case artifactLifecycleStaging, artifactLifecycleRunning: + if state.FailedAt != nil || state.PublicationTempPath != "" { + return artifactError("artifact_lifecycle_state_invalid", "validate_lifecycle_state", "", ErrArtifactState) + } + case artifactLifecyclePublishing, artifactLifecyclePublished: + if state.FailedAt != nil || + validateDelegationPublicationTempPath(delegationID, state.PublicationTempPath) != nil { + return artifactError("artifact_lifecycle_state_invalid", "validate_lifecycle_state", "", ErrArtifactState) + } + case artifactLifecycleFailed, artifactLifecycleCancelled: + if state.FailedAt == nil { + return artifactError("artifact_lifecycle_state_invalid", "validate_lifecycle_state", "", ErrArtifactState) + } + default: + return artifactError("artifact_lifecycle_state_invalid", "validate_lifecycle_state", "", ErrArtifactState) + } + if state.CallerLocation != nil { + if state.CallerLocation.Base != "workspace" && state.CallerLocation.Base != "data" { + return artifactError("artifact_lifecycle_state_invalid", "validate_lifecycle_state", "", ErrArtifactState) + } + if state.CallerLocation.RelativePath == "" { + return artifactError("artifact_lifecycle_state_invalid", "validate_lifecycle_state", "", ErrArtifactState) + } + } + if state.PublicationTempPath != "" { + if err := validateDelegationPublicationTempPath(delegationID, state.PublicationTempPath); err != nil { + return fmt.Errorf("invalid publication temp path: %w", err) + } + } + return nil +} diff --git a/internal/tools/delegation_artifacts_linux.go b/internal/tools/delegation_artifacts_linux.go new file mode 100644 index 00000000..fd642743 --- /dev/null +++ b/internal/tools/delegation_artifacts_linux.go @@ -0,0 +1,32 @@ +//go:build linux + +package tools + +import ( + "errors" + + "golang.org/x/sys/unix" +) + +func (r *artifactSecureRoot) renameNoReplace(oldName, newName string) error { + oldPath, err := validateArtifactRelativePath(oldName) + if err != nil { + return err + } + newPath, err := validateArtifactRelativePath(newName) + if err != nil { + return err + } + err = unix.Renameat2(r.fd, oldPath, r.fd, newPath, unix.RENAME_NOREPLACE) + if errors.Is(err, unix.EEXIST) { + return errors.Join(ErrArtifactPublishConflict, err) + } + if errors.Is(err, unix.ENOSYS) || errors.Is(err, unix.EINVAL) { + return errors.Join(ErrArtifactSecureUnavailable, err) + } + return err +} + +func isArtifactNotExist(err error) bool { + return errors.Is(err, unix.ENOENT) +} diff --git a/internal/tools/delegation_artifacts_posix.go b/internal/tools/delegation_artifacts_posix.go new file mode 100644 index 00000000..ade874fc --- /dev/null +++ b/internal/tools/delegation_artifacts_posix.go @@ -0,0 +1,390 @@ +//go:build linux || darwin + +package tools + +import ( + "errors" + "io" + "io/fs" + "os" + "path" + "strings" + + "golang.org/x/sys/unix" +) + +type artifactSecureRoot struct { + fd int +} + +func openArtifactSecureRoot(hostPath string) (*artifactSecureRoot, error) { + fd, err := unix.Open(hostPath, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0) + if err != nil { + return nil, translateArtifactPOSIXError(err) + } + return &artifactSecureRoot{fd: fd}, nil +} + +func (r *artifactSecureRoot) close() error { + if r == nil || r.fd < 0 { + return nil + } + fd := r.fd + r.fd = -1 + return unix.Close(fd) +} + +func (r *artifactSecureRoot) mkdirAll(relativePath string, mode fs.FileMode) error { + components, err := artifactPathComponents(relativePath) + if err != nil { + return err + } + current, err := unix.Dup(r.fd) + if err != nil { + return err + } + unix.CloseOnExec(current) + defer func() { + _ = unix.Close(current) + }() + for _, component := range components { + if err := unix.Mkdirat(current, component, uint32(mode.Perm())); err != nil && !errors.Is(err, unix.EEXIST) { + return translateArtifactPOSIXError(err) + } + next, err := unix.Openat( + current, + component, + unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, + 0, + ) + if err != nil { + return translateArtifactPOSIXError(err) + } + unix.Close(current) + current = next + } + return nil +} + +func (r *artifactSecureRoot) createSubroot(relativePath string, mode fs.FileMode) (*artifactSecureRoot, error) { + parent, name, err := r.openParent(relativePath, false) + if err != nil { + return nil, err + } + defer unix.Close(parent) + if err := unix.Mkdirat(parent, name, uint32(mode.Perm())); err != nil { + return nil, translateArtifactPOSIXError(err) + } + fd, err := unix.Openat( + parent, + name, + unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, + 0, + ) + if err != nil { + return nil, translateArtifactPOSIXError(err) + } + return &artifactSecureRoot{fd: fd}, nil +} + +func (r *artifactSecureRoot) openSubroot(relativePath string) (*artifactSecureRoot, error) { + fd, err := r.openDir(relativePath) + if err != nil { + return nil, err + } + return &artifactSecureRoot{fd: fd}, nil +} + +func (r *artifactSecureRoot) createFile( + relativePath string, + mode fs.FileMode, +) (*os.File, error) { + parent, name, err := r.openParent(relativePath, false) + if err != nil { + return nil, err + } + defer unix.Close(parent) + fd, err := unix.Openat( + parent, + name, + unix.O_WRONLY|unix.O_CREAT|unix.O_EXCL|unix.O_NOFOLLOW|unix.O_CLOEXEC, + uint32(mode.Perm()), + ) + if err != nil { + return nil, translateArtifactPOSIXError(err) + } + return os.NewFile(uintptr(fd), name), nil +} + +func (r *artifactSecureRoot) openEntry(relativePath string) (*artifactSecureEntry, error) { + parent, name, err := r.openParent(relativePath, false) + if err != nil { + return nil, err + } + defer unix.Close(parent) + fd, err := unix.Openat( + parent, + name, + unix.O_RDONLY|unix.O_NONBLOCK|unix.O_NOFOLLOW|unix.O_CLOEXEC, + 0, + ) + if err != nil { + return nil, translateArtifactPOSIXError(err) + } + var stat unix.Stat_t + if err := unix.Fstat(fd, &stat); err != nil { + unix.Close(fd) + return nil, err + } + kind := artifactEntryOther + switch stat.Mode & unix.S_IFMT { + case unix.S_IFREG: + kind = artifactEntryRegular + case unix.S_IFDIR: + kind = artifactEntryDirectory + } + return &artifactSecureEntry{ + file: os.NewFile(uintptr(fd), name), + kind: kind, + size: stat.Size, + links: uint64(stat.Nlink), + }, nil +} + +func (r *artifactSecureRoot) readDir(relativePath string) ([]string, error) { + entry, err := r.openEntry(relativePath) + if err != nil { + return nil, err + } + defer entry.close() + if entry.kind != artifactEntryDirectory { + return nil, ErrArtifactNonRegular + } + + names := make([]string, 0, artifactSecureReadBatchSize) + for { + dirEntries, readErr := entry.readDirBatch() + if readErr != nil && !errors.Is(readErr, io.EOF) { + return nil, readErr + } + if len(dirEntries) > artifactSecureMaxDirectoryEntries-len(names) { + return nil, ErrArtifactLimitExceeded + } + for _, dirEntry := range dirEntries { + names = append(names, dirEntry.Name()) + } + if errors.Is(readErr, io.EOF) { + return names, nil + } + } +} + +func (r *artifactSecureRoot) exists(relativePath string) (bool, error) { + parent, name, err := r.openParent(relativePath, false) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return false, nil + } + return false, err + } + defer unix.Close(parent) + var stat unix.Stat_t + err = unix.Fstatat(parent, name, &stat, unix.AT_SYMLINK_NOFOLLOW) + switch { + case err == nil: + return true, nil + case errors.Is(err, unix.ENOENT): + return false, nil + default: + return false, translateArtifactPOSIXError(err) + } +} + +func (r *artifactSecureRoot) syncDir(relativePath string) error { + fd, err := r.openDir(relativePath) + if err != nil { + return err + } + defer unix.Close(fd) + err = unix.Fsync(fd) + if errors.Is(err, unix.EINVAL) || errors.Is(err, unix.ENOTSUP) { + return nil + } + return err +} + +func (r *artifactSecureRoot) removeTree(relativePath string) error { + parent, name, err := r.openParent(relativePath, false) + if err != nil { + return err + } + defer unix.Close(parent) + remaining := artifactSecureCleanupEntryBudget + return removeArtifactTreeAt(parent, name, 0, &remaining) +} + +func removeArtifactTreeAt(parent int, name string, depth int, remaining *int) error { + if depth > artifactSecureMaxDepth { + return ErrArtifactLimitExceeded + } + if *remaining <= 0 { + return ErrArtifactLimitExceeded + } + *remaining-- + var stat unix.Stat_t + if err := unix.Fstatat(parent, name, &stat, unix.AT_SYMLINK_NOFOLLOW); err != nil { + if errors.Is(err, unix.ENOENT) { + return nil + } + return translateArtifactPOSIXError(err) + } + if stat.Mode&unix.S_IFMT != unix.S_IFDIR { + return translateArtifactPOSIXError(unix.Unlinkat(parent, name, 0)) + } + + fd, err := unix.Openat( + parent, + name, + unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, + 0, + ) + if err != nil { + return translateArtifactPOSIXError(err) + } + dir := os.NewFile(uintptr(fd), name) + for { + entries, readErr := dir.ReadDir(artifactSecureReadBatchSize) + if readErr != nil && !errors.Is(readErr, io.EOF) { + _ = dir.Close() + return readErr + } + for _, entry := range entries { + if err := removeArtifactTreeAt(fd, entry.Name(), depth+1, remaining); err != nil { + _ = dir.Close() + return err + } + } + if errors.Is(readErr, io.EOF) { + break + } + } + if err := dir.Close(); err != nil { + return err + } + return translateArtifactPOSIXError(unix.Unlinkat(parent, name, unix.AT_REMOVEDIR)) +} + +func (r *artifactSecureRoot) renameReplace(oldName, newName string) error { + oldPath, err := validateArtifactRelativePath(oldName) + if err != nil { + return err + } + newPath, err := validateArtifactRelativePath(newName) + if err != nil { + return err + } + if path.Dir(oldPath) != "." || path.Dir(newPath) != "." { + return ErrArtifactInvalidPath + } + return translateArtifactPOSIXError(unix.Renameat(r.fd, oldPath, r.fd, newPath)) +} + +func (r *artifactSecureRoot) openParent( + relativePath string, + create bool, +) (int, string, error) { + components, err := artifactPathComponents(relativePath) + if err != nil { + return -1, "", err + } + if len(components) == 0 { + return -1, "", ErrArtifactInvalidPath + } + parentComponents := components[:len(components)-1] + current, err := unix.Dup(r.fd) + if err != nil { + return -1, "", err + } + unix.CloseOnExec(current) + for _, component := range parentComponents { + if create { + if err := unix.Mkdirat(current, component, 0750); err != nil && !errors.Is(err, unix.EEXIST) { + unix.Close(current) + return -1, "", translateArtifactPOSIXError(err) + } + } + next, err := unix.Openat( + current, + component, + unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, + 0, + ) + if err != nil { + unix.Close(current) + return -1, "", translateArtifactPOSIXError(err) + } + unix.Close(current) + current = next + } + return current, components[len(components)-1], nil +} + +func (r *artifactSecureRoot) openDir(relativePath string) (int, error) { + if relativePath == "." { + fd, err := unix.Dup(r.fd) + if err == nil { + unix.CloseOnExec(fd) + } + return fd, err + } + components, err := artifactPathComponents(relativePath) + if err != nil { + return -1, err + } + current, err := unix.Dup(r.fd) + if err != nil { + return -1, err + } + unix.CloseOnExec(current) + for _, component := range components { + next, err := unix.Openat( + current, + component, + unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, + 0, + ) + if err != nil { + unix.Close(current) + return -1, translateArtifactPOSIXError(err) + } + unix.Close(current) + current = next + } + return current, nil +} + +func artifactPathComponents(relativePath string) ([]string, error) { + if relativePath == "." { + return nil, nil + } + clean, err := validateArtifactRelativePath(relativePath) + if err != nil { + return nil, err + } + return strings.Split(clean, "/"), nil +} + +func translateArtifactPOSIXError(err error) error { + if errors.Is(err, unix.ELOOP) { + return fmtArtifactJoined(errArtifactSymlink, err) + } + return err +} + +func fmtArtifactJoined(classification, cause error) error { + return errors.Join(classification, cause) +} + +func isArtifactAlreadyExists(err error) bool { + return errors.Is(err, unix.EEXIST) || errors.Is(err, ErrArtifactPublishConflict) +} diff --git a/internal/tools/delegation_artifacts_publish.go b/internal/tools/delegation_artifacts_publish.go new file mode 100644 index 00000000..6b8b1e8c --- /dev/null +++ b/internal/tools/delegation_artifacts_publish.go @@ -0,0 +1,475 @@ +package tools + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "path" + "sort" + "strings" + "time" + + "github.com/google/uuid" +) + +// Keep logical output paths comfortably below the strictest supported host +// path ceilings, leaving room for the captured root and publication staging. +const artifactOutputMaxPathBytes = 768 + +// Publish validates outputs and copies them into a sibling temporary directory +// beneath the captured caller root. One atomic no-replace rename makes the +// complete outputs tree and manifest visible together. +func (e *DelegationArtifactExchange) Publish( + ctx context.Context, + destination *DelegationArtifactRoot, + publishedAt time.Time, +) (DelegationArtifactPublication, error) { + return e.publish(ctx, destination, publishedAt, nil) +} + +// publishWithPreparation records the exact sibling staging path before any +// publication data is created. The callback must durably persist that path. +func (e *DelegationArtifactExchange) publishWithPreparation( + ctx context.Context, + destination *DelegationArtifactRoot, + publishedAt time.Time, + onPrepared func(string) error, +) (DelegationArtifactPublication, error) { + return e.publish(ctx, destination, publishedAt, onPrepared) +} + +func (e *DelegationArtifactExchange) publish( + ctx context.Context, + destination *DelegationArtifactRoot, + publishedAt time.Time, + onPrepared func(string) error, +) (DelegationArtifactPublication, error) { + if destination == nil || destination.root == nil { + return DelegationArtifactPublication{}, artifactError( + "artifact_missing_destination_root", "publish", "", ErrArtifactState, + ) + } + e.mu.Lock() + defer e.mu.Unlock() + if e.state != artifactExchangeOpen { + return DelegationArtifactPublication{}, artifactError( + "artifact_invalid_state", "publish", "", ErrArtifactState, + ) + } + + publication, err := e.publishLocked(ctx, destination.root, publishedAt, onPrepared) + if err != nil { + e.retainFailureLocked(time.Now(), artifactErrorCode(err)) + return DelegationArtifactPublication{}, err + } + e.state = artifactExchangePublished + return publication, nil +} + +func (e *DelegationArtifactExchange) publishLocked( + ctx context.Context, + destinationRoot *artifactSecureRoot, + publishedAt time.Time, + onPrepared func(string) error, +) (publication DelegationArtifactPublication, returnErr error) { + if err := destinationRoot.mkdirAll(".delegations", 0750); err != nil { + return DelegationArtifactPublication{}, classifyPublicationFilesystemError( + "open_publication_parent", "", err, + ) + } + publicationParent, err := destinationRoot.openSubroot(".delegations") + if err != nil { + return DelegationArtifactPublication{}, classifyPublicationFilesystemError( + "open_publication_parent", "", err, + ) + } + defer publicationParent.close() + + finalName := e.delegationID.String() + if exists, err := publicationParent.exists(finalName); err != nil { + return DelegationArtifactPublication{}, wrapArtifactFilesystemError( + "check_publication_destination", "", err, + ) + } else if exists { + return DelegationArtifactPublication{}, artifactError( + "artifact_publish_conflict", "publish", "", ErrArtifactPublishConflict, + ) + } + + tempName := ".tmp-" + finalName + "-" + uuid.NewString() + tempPath := path.Join(".delegations", tempName) + e.publicationTempPath = tempPath + if onPrepared != nil { + if err := onPrepared(tempPath); err != nil { + return DelegationArtifactPublication{}, err + } + } + tempRoot, err := publicationParent.createSubroot(tempName, 0700) + if err != nil { + return DelegationArtifactPublication{}, classifyPublicationFilesystemError( + "create_publication_temp", "", err, + ) + } + tempClosed := false + defer func() { + if !tempClosed { + if closeErr := tempRoot.close(); returnErr == nil && closeErr != nil { + returnErr = artifactError( + "artifact_close_failed", "close_publication_temp", "", closeErr, + ) + } + tempClosed = true + } + }() + if err := tempRoot.mkdirAll("outputs", 0750); err != nil { + return DelegationArtifactPublication{}, wrapArtifactFilesystemError( + "create_publication_outputs", "", err, + ) + } + + outputPaths, err := enumerateArtifactOutputs( + ctx, + e.root, + "outputs", + e.limits.MaxFiles-e.inputCount, + ) + if err != nil { + return DelegationArtifactPublication{}, err + } + outputs := make([]DelegationArtifactManifestOutput, 0, len(outputPaths)) + outputBytes := int64(0) + directories := map[string]struct{}{"outputs": {}} + for _, outputPath := range outputPaths { + if err := checkArtifactContext(ctx); err != nil { + return DelegationArtifactPublication{}, err + } + if err := validateManifestOutputPath(outputPath); err != nil { + return DelegationArtifactPublication{}, err + } + remaining := e.limits.MaxTotalBytes - e.inputBytes - outputBytes + destinationPath := outputPath + parent := path.Dir(destinationPath) + if parent != "." { + if err := tempRoot.mkdirAll(parent, 0750); err != nil { + return DelegationArtifactPublication{}, wrapArtifactFilesystemError( + "create_output_parent", destinationPath, err, + ) + } + for dir := parent; dir != "." && dir != "/"; dir = path.Dir(dir) { + directories[dir] = struct{}{} + } + } + artifact, err := copyArtifactFile( + ctx, + e.root, + outputPath, + tempRoot, + destinationPath, + e.limits.MaxFileBytes, + remaining, + 0640, + ) + if err != nil { + return DelegationArtifactPublication{}, err + } + artifact.Status = DelegationArtifactPublished + outputBytes += artifact.SizeBytes + outputs = append(outputs, DelegationArtifactManifestOutput{ + Path: artifact.Path, + SizeBytes: artifact.SizeBytes, + SHA256: artifact.SHA256, + MediaType: artifact.MediaType, + }) + } + + sort.Slice(outputs, func(i, j int) bool { return outputs[i].Path < outputs[j].Path }) + manifest := DelegationArtifactManifest{ + SchemaVersion: DelegationArtifactManifestVersion, + DelegationID: finalName, + PublishedAt: publishedAt.UTC(), + OutputCount: len(outputs), + OutputBytes: outputBytes, + Outputs: outputs, + } + if manifest.Outputs == nil { + manifest.Outputs = []DelegationArtifactManifestOutput{} + } + manifestBytes, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return DelegationArtifactPublication{}, artifactError( + "artifact_manifest_failed", "marshal_manifest", "", err, + ) + } + manifestBytes = append(manifestBytes, '\n') + manifestFile, err := tempRoot.createFile("manifest.json", 0600) + if err != nil { + return DelegationArtifactPublication{}, wrapArtifactFilesystemError( + "create_manifest", "manifest.json", err, + ) + } + if _, err := manifestFile.Write(manifestBytes); err != nil { + manifestFile.Close() + return DelegationArtifactPublication{}, artifactError( + "artifact_manifest_failed", "write_manifest", "manifest.json", err, + ) + } + if err := manifestFile.Sync(); err != nil { + manifestFile.Close() + return DelegationArtifactPublication{}, artifactError( + "artifact_sync_failed", "sync_manifest", "manifest.json", err, + ) + } + if err := manifestFile.Chmod(0640); err != nil { + manifestFile.Close() + return DelegationArtifactPublication{}, artifactError( + "artifact_chmod_failed", "protect_manifest", "manifest.json", err, + ) + } + if err := manifestFile.Close(); err != nil { + return DelegationArtifactPublication{}, artifactError( + "artifact_close_failed", "close_manifest", "manifest.json", err, + ) + } + + syncDirs := make([]string, 0, len(directories)) + for dir := range directories { + syncDirs = append(syncDirs, dir) + } + sort.Slice(syncDirs, func(i, j int) bool { + return strings.Count(syncDirs[i], "/") > strings.Count(syncDirs[j], "/") + }) + for _, dir := range syncDirs { + if err := tempRoot.syncDir(dir); err != nil { + return DelegationArtifactPublication{}, artifactError( + "artifact_sync_failed", "sync_directory", dir, err, + ) + } + } + if err := tempRoot.syncDir("."); err != nil { + return DelegationArtifactPublication{}, artifactError( + "artifact_sync_failed", "sync_publication_temp", "", err, + ) + } + if err := tempRoot.close(); err != nil { + return DelegationArtifactPublication{}, artifactError( + "artifact_close_failed", "close_publication_temp", "", err, + ) + } + tempClosed = true + if err := publicationParent.renameNoReplace(tempName, finalName); err != nil { + return DelegationArtifactPublication{}, classifyPublicationFilesystemError( + "publish", "", err, + ) + } + if err := publicationParent.syncDir("."); err != nil { + // The final publication is already atomically visible and rollback is + // unsafe. Returning failure here would invite a duplicate delegation + // even though the caller can already consume the output. + slog.Warn("delegate.artifact_publication_parent_sync_failed", + "delegation_id", e.delegationID, + "error", err, + ) + } + + rootPath := path.Join(".delegations", finalName) + return DelegationArtifactPublication{ + RootPath: rootPath, + ManifestPath: path.Join(rootPath, "manifest.json"), + Manifest: manifest, + }, nil +} + +func enumerateArtifactOutputs( + ctx context.Context, + root *artifactSecureRoot, + base string, + maxFiles int, +) ([]string, error) { + if maxFiles < 0 { + return nil, artifactError( + "artifact_file_limit", "enumerate_outputs", "", ErrArtifactLimitExceeded, + ) + } + // The configured file budget remains authoritative. Directory-only trees + // receive a matching budget, capped at the default artifact file ceiling. + maxDirectories := min(maxFiles, DelegationArtifactMaxFiles) + maxDepth := min(maxDirectories, artifactSecureMaxDepth) + maxEntries := maxFiles + maxInt := int(^uint(0) >> 1) + if maxDirectories <= maxInt-maxEntries { + maxEntries += maxDirectories + } else { + maxEntries = maxInt + } + + var outputs []string + type outputDirectory struct { + path string + depth int + } + directories := []outputDirectory{{path: base}} + entriesSeen := 0 + directoriesSeen := 0 + + for directoryIndex := 0; directoryIndex < len(directories); directoryIndex++ { + if err := checkArtifactContext(ctx); err != nil { + return nil, err + } + directory := directories[directoryIndex] + err := func() error { + entry, err := root.openEntry(directory.path) + if err != nil { + return classifyArtifactOpenError("enumerate_outputs", directory.path, err) + } + defer entry.close() + if entry.kind != artifactEntryDirectory { + return artifactError( + "artifact_non_regular", + "enumerate_outputs", + directory.path, + ErrArtifactNonRegular, + ) + } + + for { + dirEntries, readErr := entry.readDirBatch() + if readErr != nil && !errors.Is(readErr, io.EOF) { + return classifyArtifactOpenError( + "enumerate_outputs", + directory.path, + readErr, + ) + } + for _, dirEntry := range dirEntries { + if err := checkArtifactContext(ctx); err != nil { + return err + } + entriesSeen++ + if entriesSeen > maxEntries { + return artifactError( + "artifact_entry_limit", + "enumerate_outputs", + directory.path, + ErrArtifactLimitExceeded, + ) + } + name := dirEntry.Name() + if len(directory.path)+1+len(name) > artifactOutputMaxPathBytes { + return artifactError( + "artifact_path_limit", + "enumerate_outputs", + directory.path, + ErrArtifactLimitExceeded, + ) + } + logicalPath := path.Join(directory.path, name) + child, err := root.openEntry(logicalPath) + if err != nil { + return classifyArtifactOpenError("open_output", logicalPath, err) + } + switch child.kind { + case artifactEntryDirectory: + if err := child.close(); err != nil { + return wrapArtifactFilesystemError( + "close_output", + logicalPath, + err, + ) + } + directoriesSeen++ + if directoriesSeen > maxDirectories { + return artifactError( + "artifact_directory_limit", + "enumerate_outputs", + logicalPath, + ErrArtifactLimitExceeded, + ) + } + childDepth := directory.depth + 1 + if childDepth > maxDepth { + return artifactError( + "artifact_depth_limit", + "enumerate_outputs", + logicalPath, + ErrArtifactLimitExceeded, + ) + } + directories = append(directories, outputDirectory{ + path: logicalPath, + depth: childDepth, + }) + case artifactEntryRegular: + links := child.links + if err := child.close(); err != nil { + return wrapArtifactFilesystemError( + "close_output", + logicalPath, + err, + ) + } + if links != 1 { + return artifactError( + "artifact_hardlink", + "open_output", + logicalPath, + ErrArtifactHardlink, + ) + } + outputs = append(outputs, logicalPath) + if len(outputs) > maxFiles { + return artifactError( + "artifact_file_limit", + "enumerate_outputs", + logicalPath, + ErrArtifactLimitExceeded, + ) + } + default: + if err := child.close(); err != nil { + return wrapArtifactFilesystemError( + "close_output", + logicalPath, + err, + ) + } + return artifactError( + "artifact_non_regular", + "open_output", + logicalPath, + ErrArtifactNonRegular, + ) + } + } + if errors.Is(readErr, io.EOF) { + return nil + } + } + }() + if err != nil { + return nil, err + } + } + sort.Strings(outputs) + return outputs, nil +} + +func classifyPublicationFilesystemError(op, logicalPath string, err error) error { + if isArtifactAlreadyExists(err) || errors.Is(err, ErrArtifactPublishConflict) { + return artifactError("artifact_publish_conflict", op, logicalPath, ErrArtifactPublishConflict) + } + return wrapArtifactFilesystemError(op, logicalPath, err) +} + +func validateManifestOutputPath(raw string) error { + clean, err := validateArtifactRelativePath(raw) + if err != nil { + return err + } + if clean == "outputs" || !strings.HasPrefix(clean, "outputs/") { + return fmt.Errorf("%w: manifest output is outside outputs", ErrArtifactInvalidPath) + } + return nil +} diff --git a/internal/tools/delegation_artifacts_secure.go b/internal/tools/delegation_artifacts_secure.go new file mode 100644 index 00000000..ef8b2388 --- /dev/null +++ b/internal/tools/delegation_artifacts_secure.go @@ -0,0 +1,48 @@ +package tools + +import ( + "errors" + "io/fs" + "os" +) + +const ( + artifactSecureReadBatchSize = 64 + artifactSecureMaxDirectoryEntries = 4096 + artifactSecureMaxDepth = 64 + artifactSecureCleanupEntryBudget = 4096 +) + +var ( + errArtifactSymlink = errors.New("symlink rejected") + errArtifactReparsePoint = errors.New("reparse point rejected") +) + +type artifactEntryKind uint8 + +const ( + artifactEntryOther artifactEntryKind = iota + artifactEntryRegular + artifactEntryDirectory +) + +type artifactSecureEntry struct { + file *os.File + kind artifactEntryKind + size int64 + links uint64 +} + +func (e *artifactSecureEntry) close() error { + if e == nil || e.file == nil { + return nil + } + return e.file.Close() +} + +func (e *artifactSecureEntry) readDirBatch() ([]fs.DirEntry, error) { + if e == nil || e.file == nil || e.kind != artifactEntryDirectory { + return nil, ErrArtifactNonRegular + } + return e.file.ReadDir(artifactSecureReadBatchSize) +} diff --git a/internal/tools/delegation_artifacts_test.go b/internal/tools/delegation_artifacts_test.go new file mode 100644 index 00000000..b16797c9 --- /dev/null +++ b/internal/tools/delegation_artifacts_test.go @@ -0,0 +1,751 @@ +package tools + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/google/uuid" +) + +func TestDelegationArtifactExchangeTenantLayoutAndInputContract(t *testing.T) { + tenantRoot := t.TempDir() + sourcePath := t.TempDir() + if err := os.MkdirAll(filepath.Join(sourcePath, "one"), 0750); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(sourcePath, "two"), 0750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(sourcePath, "one", "report.txt"), []byte("first"), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(sourcePath, "two", "report.txt"), []byte("second"), 0600); err != nil { + t.Fatal(err) + } + + tenantID := uuid.New() + delegationID := uuid.New() + exchange := newTestDelegationExchange(t, tenantRoot, tenantID, delegationID, DelegationArtifactLimits{}) + source := openTestArtifactRoot(t, sourcePath) + staged, err := exchange.StageInputs( + context.Background(), + source, + []string{"one/report.txt", "two/report.txt"}, + ) + if err != nil { + t.Fatalf("StageInputs() error = %v", err) + } + if got, want := len(staged), 2; got != want { + t.Fatalf("staged count = %d, want %d", got, want) + } + if staged[0].Path != "inputs/report.txt" || staged[1].Path != "inputs/report-2.txt" { + t.Fatalf("staged paths = %#v", []string{staged[0].Path, staged[1].Path}) + } + + wantExchangeRoot := filepath.Join( + tenantRoot, + "collaboration", + "delegations", + delegationID.String(), + ) + if got := exchange.OutputsHostPath(); got != filepath.Join(wantExchangeRoot, "outputs") { + t.Fatalf("OutputsHostPath() = %q, want %q", got, filepath.Join(wantExchangeRoot, "outputs")) + } + mount := exchange.InputsMount() + if mount.LogicalAlias != "inputs" || mount.ContainerPath != "/workspace/inputs" || !mount.ReadOnly { + t.Fatalf("InputsMount() = %#v", mount) + } + if mount.HostRoot != filepath.Join(wantExchangeRoot, "inputs") { + t.Fatalf("input host root = %q", mount.HostRoot) + } + for _, name := range []string{"report.txt", "report-2.txt"} { + info, err := os.Stat(filepath.Join(mount.HostRoot, name)) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm()&0222 != 0 { + t.Fatalf("%s mode = %o, want read-only", name, info.Mode().Perm()) + } + } +} + +func TestDelegationArtifactStageRejectsTraversalSymlinksAndHardlinks(t *testing.T) { + testCases := []struct { + name string + prepare func(t *testing.T, source string) string + wantErr error + }{ + { + name: "traversal", + prepare: func(t *testing.T, source string) string { + return "../outside.txt" + }, + wantErr: ErrArtifactInvalidPath, + }, + { + name: "absolute", + prepare: func(t *testing.T, source string) string { + return filepath.Join(source, "outside.txt") + }, + wantErr: ErrArtifactInvalidPath, + }, + { + name: "alternate data stream", + prepare: func(t *testing.T, source string) string { + return "report.txt:secret" + }, + wantErr: ErrArtifactInvalidPath, + }, + { + name: "reserved device", + prepare: func(t *testing.T, source string) string { + return "NUL.txt" + }, + wantErr: ErrArtifactInvalidPath, + }, + { + name: "control character", + prepare: func(t *testing.T, source string) string { + return "report\n.txt" + }, + wantErr: ErrArtifactInvalidPath, + }, + { + name: "symlink", + prepare: func(t *testing.T, source string) string { + if runtime.GOOS == "windows" { + t.Skip("symlink creation requires privileges on some Windows hosts") + } + if err := os.WriteFile(filepath.Join(source, "real.txt"), []byte("secret"), 0600); err != nil { + t.Fatal(err) + } + if err := os.Symlink("real.txt", filepath.Join(source, "link.txt")); err != nil { + t.Fatal(err) + } + return "link.txt" + }, + wantErr: ErrArtifactNonRegular, + }, + { + name: "hardlink", + prepare: func(t *testing.T, source string) string { + if err := os.WriteFile(filepath.Join(source, "real.txt"), []byte("secret"), 0600); err != nil { + t.Fatal(err) + } + if err := os.Link(filepath.Join(source, "real.txt"), filepath.Join(source, "hard.txt")); err != nil { + t.Skipf("hardlinks unavailable: %v", err) + } + return "hard.txt" + }, + wantErr: ErrArtifactHardlink, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + sourcePath := t.TempDir() + input := testCase.prepare(t, sourcePath) + exchange := newTestDelegationExchange( + t, + t.TempDir(), + uuid.New(), + uuid.New(), + DelegationArtifactLimits{}, + ) + source := openTestArtifactRoot(t, sourcePath) + _, err := exchange.StageInputs(context.Background(), source, []string{input}) + if !errors.Is(err, testCase.wantErr) { + t.Fatalf("StageInputs() error = %v, want errors.Is(%v)", err, testCase.wantErr) + } + retention, retained := exchange.FailureRetention() + if !retained || retention.ReasonCode == "" || !retention.RetainUntil.After(retention.FailedAt) { + t.Fatalf("failure retention = %#v, %v", retention, retained) + } + }) + } +} + +func TestDelegationArtifactPublishManifestV1IsSortedAndPathSafe(t *testing.T) { + exchange := newTestDelegationExchange( + t, + t.TempDir(), + uuid.New(), + uuid.New(), + DelegationArtifactLimits{}, + ) + if err := os.MkdirAll(filepath.Join(exchange.OutputsHostPath(), "nested"), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(exchange.OutputsHostPath(), "z.txt"), []byte("z"), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(exchange.OutputsHostPath(), "nested", "a.txt"), []byte("alpha"), 0600); err != nil { + t.Fatal(err) + } + + destinationPath := t.TempDir() + destination := openTestArtifactRoot(t, destinationPath) + publishedAt := time.Date(2026, 7, 29, 8, 30, 0, 0, time.FixedZone("ICT", 7*60*60)) + publication, err := exchange.Publish(context.Background(), destination, publishedAt) + if err != nil { + t.Fatalf("Publish() error = %v", err) + } + if publication.RootPath != filepath.ToSlash(filepath.Join(".delegations", exchange.DelegationID().String())) { + t.Fatalf("publication root = %q", publication.RootPath) + } + if publication.Manifest.SchemaVersion != 1 || + publication.Manifest.DelegationID != exchange.DelegationID().String() || + publication.Manifest.PublishedAt.Location() != time.UTC || + publication.Manifest.OutputCount != 2 || + publication.Manifest.OutputBytes != 6 { + t.Fatalf("manifest = %#v", publication.Manifest) + } + if got := []string{ + publication.Manifest.Outputs[0].Path, + publication.Manifest.Outputs[1].Path, + }; got[0] != "outputs/nested/a.txt" || got[1] != "outputs/z.txt" { + t.Fatalf("manifest output order = %#v", got) + } + for _, output := range publication.Manifest.Outputs { + if err := validateManifestOutputPath(output.Path); err != nil { + t.Fatalf("manifest output path %q invalid: %v", output.Path, err) + } + if len(output.SHA256) != 64 || output.MediaType == "" { + t.Fatalf("manifest output = %#v", output) + } + } + + manifestBytes, err := os.ReadFile(filepath.Join(destinationPath, filepath.FromSlash(publication.ManifestPath))) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(manifestBytes), destinationPath) || + strings.Contains(string(manifestBytes), exchange.OutputsHostPath()) { + t.Fatalf("manifest leaked host path: %s", manifestBytes) + } + var durable DelegationArtifactManifest + if err := json.Unmarshal(manifestBytes, &durable); err != nil { + t.Fatalf("decode manifest: %v", err) + } + if durable.Outputs == nil || durable.OutputCount != len(durable.Outputs) { + t.Fatalf("durable outputs = %#v", durable.Outputs) + } + var rawManifest map[string]json.RawMessage + if err := json.Unmarshal(manifestBytes, &rawManifest); err != nil { + t.Fatal(err) + } + if len(rawManifest) != 6 { + t.Fatalf("manifest persisted unexpected fields: %v", rawManifest) + } + for _, forbidden := range []string{"tenant_id", "agent_id", "workspace", "trace_id", "status"} { + if _, exists := rawManifest[forbidden]; exists { + t.Fatalf("manifest persisted forbidden field %q", forbidden) + } + } +} + +func TestDelegationArtifactPublishEmptyManifest(t *testing.T) { + exchange := newTestDelegationExchange( + t, + t.TempDir(), + uuid.New(), + uuid.New(), + DelegationArtifactLimits{}, + ) + destinationPath := t.TempDir() + publication, err := exchange.Publish( + context.Background(), + openTestArtifactRoot(t, destinationPath), + time.Unix(123, 0), + ) + if err != nil { + t.Fatalf("Publish() error = %v", err) + } + if publication.Manifest.OutputCount != 0 || + publication.Manifest.OutputBytes != 0 || + publication.Manifest.Outputs == nil || + len(publication.Manifest.Outputs) != 0 { + t.Fatalf("empty manifest = %#v", publication.Manifest) + } + manifestBytes, err := os.ReadFile(filepath.Join(destinationPath, filepath.FromSlash(publication.ManifestPath))) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(manifestBytes), `"outputs": []`) { + t.Fatalf("empty outputs must encode as [], manifest: %s", manifestBytes) + } +} + +func TestDelegationArtifactPublishRejectsExcessEmptyDirectories(t *testing.T) { + exchange := newTestDelegationExchange( + t, + t.TempDir(), + uuid.New(), + uuid.New(), + DelegationArtifactLimits{}, + ) + for index := 0; index <= DelegationArtifactMaxFiles; index++ { + name := fmt.Sprintf("empty-%03d", index) + if err := os.Mkdir(filepath.Join(exchange.OutputsHostPath(), name), 0700); err != nil { + t.Fatal(err) + } + } + + _, err := exchange.Publish( + context.Background(), + openTestArtifactRoot(t, t.TempDir()), + time.Now(), + ) + var artifactErr *DelegationArtifactError + if !errors.Is(err, ErrArtifactLimitExceeded) || + !errors.As(err, &artifactErr) || + artifactErr.Code != "artifact_directory_limit" { + t.Fatalf("Publish() error = %v, want artifact_directory_limit", err) + } +} + +func TestDelegationArtifactPublishRejectsDeepOutputTree(t *testing.T) { + exchange := newTestDelegationExchange( + t, + t.TempDir(), + uuid.New(), + uuid.New(), + DelegationArtifactLimits{}, + ) + current := exchange.OutputsHostPath() + for depth := 0; depth <= artifactSecureMaxDepth; depth++ { + current = filepath.Join(current, "d") + if err := os.Mkdir(current, 0700); err != nil { + t.Fatal(err) + } + } + + _, err := exchange.Publish( + context.Background(), + openTestArtifactRoot(t, t.TempDir()), + time.Now(), + ) + var artifactErr *DelegationArtifactError + if !errors.Is(err, ErrArtifactLimitExceeded) || + !errors.As(err, &artifactErr) || + artifactErr.Code != "artifact_depth_limit" { + t.Fatalf("Publish() error = %v, want artifact_depth_limit", err) + } +} + +func TestDelegationArtifactPublishRejectsLongLogicalOutputPath(t *testing.T) { + exchange := newTestDelegationExchange( + t, + t.TempDir(), + uuid.New(), + uuid.New(), + DelegationArtifactLimits{}, + ) + currentHostPath := exchange.OutputsHostPath() + logicalPath := "outputs" + for depth := 0; len(logicalPath) <= artifactOutputMaxPathBytes; depth++ { + if depth >= artifactSecureMaxDepth { + t.Fatal("test path reached depth limit before path-length limit") + } + component := fmt.Sprintf("%02d-%s", depth, strings.Repeat("x", 47)) + currentHostPath = filepath.Join(currentHostPath, component) + logicalPath += "/" + component + if err := os.Mkdir(currentHostPath, 0700); err != nil { + t.Fatal(err) + } + } + + _, err := exchange.Publish( + context.Background(), + openTestArtifactRoot(t, t.TempDir()), + time.Now(), + ) + var artifactErr *DelegationArtifactError + if !errors.Is(err, ErrArtifactLimitExceeded) || + !errors.As(err, &artifactErr) || + artifactErr.Code != "artifact_path_limit" { + t.Fatalf("Publish() error = %v, want artifact_path_limit", err) + } +} + +func TestDelegationArtifactPublishPreservesRegularOutputFileLimit(t *testing.T) { + limits := DelegationArtifactLimits{ + MaxFileBytes: 8, + MaxTotalBytes: 32, + MaxFiles: 2, + } + exchange := newTestDelegationExchange( + t, + t.TempDir(), + uuid.New(), + uuid.New(), + limits, + ) + for _, name := range []string{"one.txt", "two.txt", "three.txt"} { + if err := os.WriteFile(filepath.Join(exchange.OutputsHostPath(), name), []byte(name), 0600); err != nil { + t.Fatal(err) + } + } + + _, err := exchange.Publish( + context.Background(), + openTestArtifactRoot(t, t.TempDir()), + time.Now(), + ) + var artifactErr *DelegationArtifactError + if !errors.Is(err, ErrArtifactLimitExceeded) || + !errors.As(err, &artifactErr) || + artifactErr.Code != "artifact_file_limit" { + t.Fatalf("Publish() error = %v, want artifact_file_limit", err) + } +} + +func TestDelegationArtifactPublishRejectsSymlinkAndHardlinkOutputs(t *testing.T) { + testCases := []struct { + name string + prepare func(t *testing.T, exchange *DelegationArtifactExchange) + wantErr error + }{ + { + name: "symlink", + prepare: func(t *testing.T, exchange *DelegationArtifactExchange) { + if runtime.GOOS == "windows" { + t.Skip("symlink creation requires privileges on some Windows hosts") + } + outside := filepath.Join(t.TempDir(), "outside.txt") + if err := os.WriteFile(outside, []byte("secret"), 0600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(exchange.OutputsHostPath(), "link.txt")); err != nil { + t.Fatal(err) + } + }, + wantErr: ErrArtifactNonRegular, + }, + { + name: "hardlink", + prepare: func(t *testing.T, exchange *DelegationArtifactExchange) { + outside := filepath.Join(t.TempDir(), "outside.txt") + if err := os.WriteFile(outside, []byte("secret"), 0600); err != nil { + t.Fatal(err) + } + if err := os.Link(outside, filepath.Join(exchange.OutputsHostPath(), "hard.txt")); err != nil { + t.Skipf("hardlinks unavailable: %v", err) + } + }, + wantErr: ErrArtifactHardlink, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + exchange := newTestDelegationExchange( + t, + t.TempDir(), + uuid.New(), + uuid.New(), + DelegationArtifactLimits{}, + ) + testCase.prepare(t, exchange) + destinationPath := t.TempDir() + _, err := exchange.Publish( + context.Background(), + openTestArtifactRoot(t, destinationPath), + time.Now(), + ) + if !errors.Is(err, testCase.wantErr) { + t.Fatalf("Publish() error = %v, want errors.Is(%v)", err, testCase.wantErr) + } + if _, statErr := os.Stat(filepath.Join(destinationPath, ".delegations", exchange.DelegationID().String())); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("final destination visible after rejection: %v", statErr) + } + retention, retained := exchange.FailureRetention() + if !retained || !strings.HasPrefix( + retention.PublicationTempPath, + ".delegations/.tmp-"+exchange.DelegationID().String()+"-", + ) { + t.Fatalf("publication temp path not retained for janitor: %#v, %v", retention, retained) + } + parentEntries, readErr := os.ReadDir(filepath.Join(destinationPath, ".delegations")) + if readErr != nil { + t.Fatal(readErr) + } + for _, entry := range parentEntries { + if !strings.HasPrefix(entry.Name(), ".tmp-") { + t.Fatalf("unexpected publication entry remains: %s", entry.Name()) + } + } + }) + } +} + +func TestDelegationArtifactPublishNoReplaceCollision(t *testing.T) { + exchange := newTestDelegationExchange( + t, + t.TempDir(), + uuid.New(), + uuid.New(), + DelegationArtifactLimits{}, + ) + if err := os.WriteFile(filepath.Join(exchange.OutputsHostPath(), "new.txt"), []byte("new"), 0600); err != nil { + t.Fatal(err) + } + destinationPath := t.TempDir() + finalPath := filepath.Join(destinationPath, ".delegations", exchange.DelegationID().String()) + if err := os.MkdirAll(finalPath, 0700); err != nil { + t.Fatal(err) + } + sentinel := filepath.Join(finalPath, "sentinel.txt") + if err := os.WriteFile(sentinel, []byte("original"), 0600); err != nil { + t.Fatal(err) + } + + _, err := exchange.Publish( + context.Background(), + openTestArtifactRoot(t, destinationPath), + time.Now(), + ) + if !errors.Is(err, ErrArtifactPublishConflict) { + t.Fatalf("Publish() error = %v, want conflict", err) + } + content, readErr := os.ReadFile(sentinel) + if readErr != nil || string(content) != "original" { + t.Fatalf("pre-existing destination changed: %q, %v", content, readErr) + } +} + +func TestDelegationArtifactSecureRenameNeverReplaces(t *testing.T) { + root := openTestArtifactRoot(t, t.TempDir()) + source, err := root.root.createSubroot("source", 0700) + if err != nil { + t.Fatal(err) + } + if err := source.close(); err != nil { + t.Fatal(err) + } + destination, err := root.root.createSubroot("destination", 0700) + if err != nil { + t.Fatal(err) + } + if err := destination.close(); err != nil { + t.Fatal(err) + } + + err = root.root.renameNoReplace("source", "destination") + if !errors.Is(err, ErrArtifactPublishConflict) { + t.Fatalf("renameNoReplace() error = %v, want conflict", err) + } + for _, name := range []string{"source", "destination"} { + exists, statErr := root.root.exists(name) + if statErr != nil || !exists { + t.Fatalf("%s existence = %v, %v; rename replaced an entry", name, exists, statErr) + } + } +} + +func TestDelegationArtifactPublishUsesCapturedDestinationRoot(t *testing.T) { + exchange := newTestDelegationExchange( + t, + t.TempDir(), + uuid.New(), + uuid.New(), + DelegationArtifactLimits{}, + ) + if err := os.WriteFile(filepath.Join(exchange.OutputsHostPath(), "result.txt"), []byte("result"), 0600); err != nil { + t.Fatal(err) + } + + parent := t.TempDir() + originalPath := filepath.Join(parent, "workspace") + movedPath := filepath.Join(parent, "workspace-captured") + if err := os.Mkdir(originalPath, 0700); err != nil { + t.Fatal(err) + } + destination := openTestArtifactRoot(t, originalPath) + if err := os.Rename(originalPath, movedPath); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(originalPath, 0700); err != nil { + t.Fatal(err) + } + + publication, err := exchange.Publish(context.Background(), destination, time.Now()) + if err != nil { + t.Fatalf("Publish() error = %v", err) + } + if _, err := os.Stat(filepath.Join(movedPath, filepath.FromSlash(publication.ManifestPath))); err != nil { + t.Fatalf("artifact not published to captured root: %v", err) + } + if _, err := os.Stat(filepath.Join(originalPath, filepath.FromSlash(publication.ManifestPath))); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("replacement path received publication: %v", err) + } +} + +func TestDelegationArtifactBudgetsAreSharedAcrossInputsAndOutputs(t *testing.T) { + limits := DelegationArtifactLimits{ + MaxFileBytes: 5, + MaxTotalBytes: 8, + MaxFiles: 2, + } + sourcePath := t.TempDir() + if err := os.WriteFile(filepath.Join(sourcePath, "input.bin"), []byte("1234"), 0600); err != nil { + t.Fatal(err) + } + exchange := newTestDelegationExchange(t, t.TempDir(), uuid.New(), uuid.New(), limits) + if _, err := exchange.StageInputs( + context.Background(), + openTestArtifactRoot(t, sourcePath), + []string{"input.bin"}, + ); err != nil { + t.Fatalf("StageInputs() error = %v", err) + } + if err := os.WriteFile(filepath.Join(exchange.OutputsHostPath(), "output.bin"), []byte("12345"), 0600); err != nil { + t.Fatal(err) + } + destinationPath := t.TempDir() + _, err := exchange.Publish( + context.Background(), + openTestArtifactRoot(t, destinationPath), + time.Now(), + ) + if !errors.Is(err, ErrArtifactLimitExceeded) { + t.Fatalf("Publish() error = %v, want shared total-byte limit", err) + } + if _, statErr := os.Stat(filepath.Join(destinationPath, ".delegations", exchange.DelegationID().String())); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("final destination visible after budget failure: %v", statErr) + } +} + +func TestDelegationArtifactInputFileLimitRejectsBeforeStaging(t *testing.T) { + limits := DelegationArtifactLimits{ + MaxFileBytes: 8, + MaxTotalBytes: 16, + MaxFiles: 1, + } + sourcePath := t.TempDir() + for _, name := range []string{"one.txt", "two.txt"} { + if err := os.WriteFile(filepath.Join(sourcePath, name), []byte(name), 0600); err != nil { + t.Fatal(err) + } + } + exchange := newTestDelegationExchange(t, t.TempDir(), uuid.New(), uuid.New(), limits) + _, err := exchange.StageInputs( + context.Background(), + openTestArtifactRoot(t, sourcePath), + []string{"one.txt", "two.txt"}, + ) + if !errors.Is(err, ErrArtifactLimitExceeded) { + t.Fatalf("StageInputs() error = %v, want file limit", err) + } + entries, readErr := os.ReadDir(exchange.InputsMount().HostRoot) + if readErr != nil { + t.Fatal(readErr) + } + if len(entries) != 0 { + t.Fatalf("staging began before structured input count rejection: %v", entries) + } +} + +func TestDelegationArtifactPublishRejectsSymlinkPublicationParent(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlink creation requires privileges on some Windows hosts") + } + exchange := newTestDelegationExchange( + t, + t.TempDir(), + uuid.New(), + uuid.New(), + DelegationArtifactLimits{}, + ) + destinationPath := t.TempDir() + outsidePath := t.TempDir() + if err := os.Symlink(outsidePath, filepath.Join(destinationPath, ".delegations")); err != nil { + t.Fatal(err) + } + _, err := exchange.Publish( + context.Background(), + openTestArtifactRoot(t, destinationPath), + time.Now(), + ) + if err == nil { + t.Fatal("Publish() accepted symlink publication parent") + } + entries, readErr := os.ReadDir(outsidePath) + if readErr != nil { + t.Fatal(readErr) + } + if len(entries) != 0 { + t.Fatalf("publication escaped through symlink: %v", entries) + } +} + +func TestDelegationArtifactFailureRetentionCleanupHook(t *testing.T) { + exchange := newTestDelegationExchange( + t, + t.TempDir(), + uuid.New(), + uuid.New(), + DelegationArtifactLimits{}, + ) + failedAt := time.Date(2026, 7, 29, 0, 0, 0, 0, time.UTC) + exchange.RetainFailure(failedAt, "validation_failed") + exchange.RetainFailure(failedAt.Add(time.Hour), "must_not_extend_ttl") + retention, ok := exchange.FailureRetention() + if !ok || retention.FailedAt != failedAt || retention.ReasonCode != "validation_failed" { + t.Fatalf("retention = %#v, %v", retention, ok) + } + if exchange.ReadyForCleanup(failedAt.Add(59 * time.Minute)) { + t.Fatal("exchange ready for cleanup before TTL") + } + if !exchange.ReadyForCleanup(failedAt.Add(60 * time.Minute)) { + t.Fatal("exchange not ready for cleanup at TTL") + } +} + +func newTestDelegationExchange( + t *testing.T, + tenantRoot string, + tenantID uuid.UUID, + delegationID uuid.UUID, + limits DelegationArtifactLimits, +) *DelegationArtifactExchange { + t.Helper() + exchange, err := NewDelegationArtifactExchange( + tenantRoot, + tenantID, + delegationID, + limits, + 0, + ) + if err != nil { + t.Fatalf("NewDelegationArtifactExchange() error = %v", err) + } + t.Cleanup(func() { + if err := exchange.Close(); err != nil { + t.Errorf("close exchange: %v", err) + } + }) + return exchange +} + +func openTestArtifactRoot(t *testing.T, hostPath string) *DelegationArtifactRoot { + t.Helper() + root, err := OpenDelegationArtifactRoot(hostPath) + if err != nil { + t.Fatalf("OpenDelegationArtifactRoot() error = %v", err) + } + t.Cleanup(func() { + if err := root.Close(); err != nil { + t.Errorf("close root: %v", err) + } + }) + return root +} diff --git a/internal/tools/delegation_artifacts_unsupported.go b/internal/tools/delegation_artifacts_unsupported.go new file mode 100644 index 00000000..ffc437e0 --- /dev/null +++ b/internal/tools/delegation_artifacts_unsupported.go @@ -0,0 +1,63 @@ +//go:build !linux && !darwin && !windows + +package tools + +import ( + "io/fs" + "os" +) + +type artifactSecureRoot struct{} + +func openArtifactSecureRoot(string) (*artifactSecureRoot, error) { + return nil, ErrArtifactSecureUnavailable +} + +func (r *artifactSecureRoot) close() error { return nil } + +func (r *artifactSecureRoot) mkdirAll(string, fs.FileMode) error { + return ErrArtifactSecureUnavailable +} + +func (r *artifactSecureRoot) createSubroot(string, fs.FileMode) (*artifactSecureRoot, error) { + return nil, ErrArtifactSecureUnavailable +} + +func (r *artifactSecureRoot) openSubroot(string) (*artifactSecureRoot, error) { + return nil, ErrArtifactSecureUnavailable +} + +func (r *artifactSecureRoot) createFile(string, fs.FileMode) (*os.File, error) { + return nil, ErrArtifactSecureUnavailable +} + +func (r *artifactSecureRoot) openEntry(string) (*artifactSecureEntry, error) { + return nil, ErrArtifactSecureUnavailable +} + +func (r *artifactSecureRoot) readDir(string) ([]string, error) { + return nil, ErrArtifactSecureUnavailable +} + +func (r *artifactSecureRoot) exists(string) (bool, error) { + return false, ErrArtifactSecureUnavailable +} + +func (r *artifactSecureRoot) syncDir(string) error { + return ErrArtifactSecureUnavailable +} + +func (r *artifactSecureRoot) removeTree(string) error { + return ErrArtifactSecureUnavailable +} + +func (r *artifactSecureRoot) renameNoReplace(string, string) error { + return ErrArtifactSecureUnavailable +} + +func (r *artifactSecureRoot) renameReplace(string, string) error { + return ErrArtifactSecureUnavailable +} + +func isArtifactAlreadyExists(error) bool { return false } +func isArtifactNotExist(error) bool { return false } diff --git a/internal/tools/delegation_artifacts_windows.go b/internal/tools/delegation_artifacts_windows.go new file mode 100644 index 00000000..84a4ec79 --- /dev/null +++ b/internal/tools/delegation_artifacts_windows.go @@ -0,0 +1,574 @@ +//go:build windows + +package tools + +import ( + "errors" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "strings" + "unsafe" + + "golang.org/x/sys/windows" +) + +type artifactSecureRoot struct { + handle windows.Handle +} + +type artifactFileRenameInformation struct { + ReplaceIfExists uint32 + RootDirectory windows.Handle + FileNameLength uint32 + FileName [1]uint16 +} + +func openArtifactSecureRoot(hostPath string) (*artifactSecureRoot, error) { + absolute, err := filepath.Abs(hostPath) + if err != nil { + return nil, err + } + handle, err := artifactNtOpen( + 0, + artifactNTAbsolutePath(absolute), + windows.FILE_GENERIC_READ|windows.FILE_GENERIC_WRITE|windows.DELETE|windows.SYNCHRONIZE, + windows.FILE_OPEN, + windows.FILE_DIRECTORY_FILE|windows.FILE_OPEN_REPARSE_POINT, + 0, + ) + if err != nil { + return nil, translateArtifactWindowsError(err) + } + if err := rejectArtifactReparsePoint(handle); err != nil { + windows.CloseHandle(handle) + return nil, err + } + return &artifactSecureRoot{handle: handle}, nil +} + +func (r *artifactSecureRoot) close() error { + if r == nil || r.handle == 0 || r.handle == windows.InvalidHandle { + return nil + } + handle := r.handle + r.handle = windows.InvalidHandle + return windows.CloseHandle(handle) +} + +func (r *artifactSecureRoot) mkdirAll(relativePath string, _ fs.FileMode) error { + components, err := artifactPathComponentsWindows(relativePath) + if err != nil { + return err + } + current, err := duplicateArtifactHandle(r.handle) + if err != nil { + return err + } + defer windows.CloseHandle(current) + for _, component := range components { + next, err := artifactNtOpen( + current, + component, + windows.FILE_GENERIC_READ|windows.FILE_GENERIC_WRITE|windows.DELETE|windows.SYNCHRONIZE, + windows.FILE_OPEN_IF, + windows.FILE_DIRECTORY_FILE|windows.FILE_OPEN_REPARSE_POINT, + windows.FILE_ATTRIBUTE_NORMAL, + ) + if err != nil { + return translateArtifactWindowsError(err) + } + if err := rejectArtifactReparsePoint(next); err != nil { + windows.CloseHandle(next) + return err + } + windows.CloseHandle(current) + current = next + } + return nil +} + +func (r *artifactSecureRoot) createSubroot(relativePath string, _ fs.FileMode) (*artifactSecureRoot, error) { + parent, name, err := r.openParent(relativePath) + if err != nil { + return nil, err + } + defer windows.CloseHandle(parent) + handle, err := artifactNtOpen( + parent, + name, + windows.FILE_GENERIC_READ|windows.FILE_GENERIC_WRITE|windows.DELETE|windows.SYNCHRONIZE, + windows.FILE_CREATE, + windows.FILE_DIRECTORY_FILE|windows.FILE_OPEN_REPARSE_POINT, + windows.FILE_ATTRIBUTE_NORMAL, + ) + if err != nil { + return nil, translateArtifactWindowsError(err) + } + if err := rejectArtifactReparsePoint(handle); err != nil { + windows.CloseHandle(handle) + return nil, err + } + return &artifactSecureRoot{handle: handle}, nil +} + +func (r *artifactSecureRoot) openSubroot(relativePath string) (*artifactSecureRoot, error) { + handle, err := r.openDir(relativePath) + if err != nil { + return nil, err + } + return &artifactSecureRoot{handle: handle}, nil +} + +func (r *artifactSecureRoot) createFile(relativePath string, _ fs.FileMode) (*os.File, error) { + parent, name, err := r.openParent(relativePath) + if err != nil { + return nil, err + } + defer windows.CloseHandle(parent) + handle, err := artifactNtOpen( + parent, + name, + windows.FILE_GENERIC_READ|windows.FILE_GENERIC_WRITE|windows.DELETE|windows.SYNCHRONIZE, + windows.FILE_CREATE, + windows.FILE_NON_DIRECTORY_FILE|windows.FILE_OPEN_REPARSE_POINT, + windows.FILE_ATTRIBUTE_NORMAL, + ) + if err != nil { + return nil, translateArtifactWindowsError(err) + } + if err := rejectArtifactReparsePoint(handle); err != nil { + windows.CloseHandle(handle) + return nil, err + } + return os.NewFile(uintptr(handle), name), nil +} + +func (r *artifactSecureRoot) openEntry(relativePath string) (*artifactSecureEntry, error) { + parent, name, err := r.openParent(relativePath) + if err != nil { + return nil, err + } + defer windows.CloseHandle(parent) + handle, err := artifactNtOpen( + parent, + name, + windows.FILE_GENERIC_READ|windows.SYNCHRONIZE, + windows.FILE_OPEN, + windows.FILE_OPEN_REPARSE_POINT, + 0, + ) + if err != nil { + return nil, translateArtifactWindowsError(err) + } + var info windows.ByHandleFileInformation + if err := windows.GetFileInformationByHandle(handle, &info); err != nil { + windows.CloseHandle(handle) + return nil, err + } + if info.FileAttributes&windows.FILE_ATTRIBUTE_REPARSE_POINT != 0 { + windows.CloseHandle(handle) + return nil, errArtifactReparsePoint + } + kind := artifactEntryRegular + if info.FileAttributes&windows.FILE_ATTRIBUTE_DIRECTORY != 0 { + kind = artifactEntryDirectory + } + size := int64(uint64(info.FileSizeHigh)<<32 | uint64(info.FileSizeLow)) + return &artifactSecureEntry{ + file: os.NewFile(uintptr(handle), name), + kind: kind, + size: size, + links: uint64(info.NumberOfLinks), + }, nil +} + +func (r *artifactSecureRoot) readDir(relativePath string) ([]string, error) { + entry, err := r.openEntry(relativePath) + if err != nil { + return nil, err + } + defer entry.close() + if entry.kind != artifactEntryDirectory { + return nil, ErrArtifactNonRegular + } + + names := make([]string, 0, artifactSecureReadBatchSize) + for { + dirEntries, readErr := entry.readDirBatch() + if readErr != nil && !errors.Is(readErr, io.EOF) { + return nil, readErr + } + if len(dirEntries) > artifactSecureMaxDirectoryEntries-len(names) { + return nil, ErrArtifactLimitExceeded + } + for _, dirEntry := range dirEntries { + names = append(names, dirEntry.Name()) + } + if errors.Is(readErr, io.EOF) { + return names, nil + } + } +} + +func (r *artifactSecureRoot) exists(relativePath string) (bool, error) { + parent, name, err := r.openParent(relativePath) + if err != nil { + if isArtifactWindowsNotExist(err) { + return false, nil + } + return false, err + } + defer windows.CloseHandle(parent) + handle, err := artifactNtOpen( + parent, + name, + windows.FILE_READ_ATTRIBUTES|windows.SYNCHRONIZE, + windows.FILE_OPEN, + windows.FILE_OPEN_REPARSE_POINT, + 0, + ) + if err != nil { + if isArtifactWindowsNotExist(err) { + return false, nil + } + return false, translateArtifactWindowsError(err) + } + windows.CloseHandle(handle) + return true, nil +} + +func (r *artifactSecureRoot) syncDir(relativePath string) error { + handle, err := r.openDir(relativePath) + if err != nil { + return err + } + defer windows.CloseHandle(handle) + err = windows.FlushFileBuffers(handle) + if errors.Is(err, windows.ERROR_ACCESS_DENIED) || errors.Is(err, windows.ERROR_INVALID_HANDLE) { + return nil + } + return err +} + +func (r *artifactSecureRoot) removeTree(relativePath string) error { + parent, name, err := r.openParent(relativePath) + if err != nil { + return err + } + defer windows.CloseHandle(parent) + remaining := artifactSecureCleanupEntryBudget + return removeArtifactTreeAt(parent, name, 0, &remaining) +} + +func removeArtifactTreeAt(parent windows.Handle, name string, depth int, remaining *int) error { + if depth > artifactSecureMaxDepth { + return ErrArtifactLimitExceeded + } + if *remaining <= 0 { + return ErrArtifactLimitExceeded + } + *remaining-- + handle, err := artifactNtOpen( + parent, + name, + windows.FILE_GENERIC_READ|windows.DELETE|windows.SYNCHRONIZE, + windows.FILE_OPEN, + windows.FILE_OPEN_REPARSE_POINT, + 0, + ) + if err != nil { + if isArtifactWindowsNotExist(err) { + return nil + } + return translateArtifactWindowsError(err) + } + defer windows.CloseHandle(handle) + + var info windows.ByHandleFileInformation + if err := windows.GetFileInformationByHandle(handle, &info); err != nil { + return err + } + isDirectory := info.FileAttributes&windows.FILE_ATTRIBUTE_DIRECTORY != 0 + isReparsePoint := info.FileAttributes&windows.FILE_ATTRIBUTE_REPARSE_POINT != 0 + if isDirectory && !isReparsePoint { + duplicate, err := duplicateArtifactHandle(handle) + if err != nil { + return err + } + dir := os.NewFile(uintptr(duplicate), name) + for { + entries, readErr := dir.ReadDir(artifactSecureReadBatchSize) + if readErr != nil && !errors.Is(readErr, io.EOF) { + _ = dir.Close() + return readErr + } + for _, entry := range entries { + if err := removeArtifactTreeAt(handle, entry.Name(), depth+1, remaining); err != nil { + _ = dir.Close() + return err + } + } + if errors.Is(readErr, io.EOF) { + break + } + } + if err := dir.Close(); err != nil { + return err + } + } + deleteFlag := byte(1) + return windows.SetFileInformationByHandle( + handle, + windows.FileDispositionInfo, + &deleteFlag, + 1, + ) +} + +func (r *artifactSecureRoot) renameNoReplace(oldName, newName string) error { + return r.rename(oldName, newName, false) +} + +func (r *artifactSecureRoot) renameReplace(oldName, newName string) error { + return r.rename(oldName, newName, true) +} + +func (r *artifactSecureRoot) rename(oldName, newName string, replace bool) error { + oldPath, err := validateArtifactRelativePath(oldName) + if err != nil { + return err + } + newPath, err := validateArtifactRelativePath(newName) + if err != nil { + return err + } + if path.Dir(oldPath) != "." || path.Dir(newPath) != "." { + return ErrArtifactInvalidPath + } + source, err := artifactNtOpen( + r.handle, + oldPath, + windows.FILE_GENERIC_READ|windows.DELETE|windows.SYNCHRONIZE, + windows.FILE_OPEN, + windows.FILE_OPEN_REPARSE_POINT, + 0, + ) + if err != nil { + return translateArtifactWindowsError(err) + } + defer windows.CloseHandle(source) + if err := rejectArtifactReparsePoint(source); err != nil { + return err + } + + newNameUTF16, err := windows.UTF16FromString(newPath) + if err != nil { + return err + } + nameBytes := (len(newNameUTF16) - 1) * 2 + var layout artifactFileRenameInformation + bufferSize := int(unsafe.Offsetof(layout.FileName)) + nameBytes + buffer := make([]byte, bufferSize) + info := (*artifactFileRenameInformation)(unsafe.Pointer(&buffer[0])) + if replace { + info.ReplaceIfExists = 1 + } + info.RootDirectory = r.handle + info.FileNameLength = uint32(nameBytes) + copy( + (*[windows.MAX_LONG_PATH]uint16)(unsafe.Pointer(&info.FileName[0]))[:nameBytes/2:nameBytes/2], + newNameUTF16[:len(newNameUTF16)-1], + ) + var iosb windows.IO_STATUS_BLOCK + err = windows.NtSetInformationFile( + source, + &iosb, + &buffer[0], + uint32(bufferSize), + windows.FileRenameInformation, + ) + if !replace && isArtifactAlreadyExists(err) { + return errors.Join(ErrArtifactPublishConflict, err) + } + return translateArtifactWindowsError(err) +} + +func (r *artifactSecureRoot) openParent(relativePath string) (windows.Handle, string, error) { + components, err := artifactPathComponentsWindows(relativePath) + if err != nil { + return windows.InvalidHandle, "", err + } + if len(components) == 0 { + return windows.InvalidHandle, "", ErrArtifactInvalidPath + } + current, err := duplicateArtifactHandle(r.handle) + if err != nil { + return windows.InvalidHandle, "", err + } + for _, component := range components[:len(components)-1] { + next, err := artifactNtOpen( + current, + component, + windows.FILE_GENERIC_READ|windows.FILE_GENERIC_WRITE|windows.DELETE|windows.SYNCHRONIZE, + windows.FILE_OPEN, + windows.FILE_DIRECTORY_FILE|windows.FILE_OPEN_REPARSE_POINT, + 0, + ) + if err != nil { + windows.CloseHandle(current) + return windows.InvalidHandle, "", translateArtifactWindowsError(err) + } + if err := rejectArtifactReparsePoint(next); err != nil { + windows.CloseHandle(next) + windows.CloseHandle(current) + return windows.InvalidHandle, "", err + } + windows.CloseHandle(current) + current = next + } + return current, components[len(components)-1], nil +} + +func (r *artifactSecureRoot) openDir(relativePath string) (windows.Handle, error) { + if relativePath == "." { + return duplicateArtifactHandle(r.handle) + } + components, err := artifactPathComponentsWindows(relativePath) + if err != nil { + return windows.InvalidHandle, err + } + current, err := duplicateArtifactHandle(r.handle) + if err != nil { + return windows.InvalidHandle, err + } + for _, component := range components { + next, err := artifactNtOpen( + current, + component, + windows.FILE_GENERIC_READ|windows.FILE_GENERIC_WRITE|windows.DELETE|windows.SYNCHRONIZE, + windows.FILE_OPEN, + windows.FILE_DIRECTORY_FILE|windows.FILE_OPEN_REPARSE_POINT, + 0, + ) + if err != nil { + windows.CloseHandle(current) + return windows.InvalidHandle, translateArtifactWindowsError(err) + } + if err := rejectArtifactReparsePoint(next); err != nil { + windows.CloseHandle(next) + windows.CloseHandle(current) + return windows.InvalidHandle, err + } + windows.CloseHandle(current) + current = next + } + return current, nil +} + +func artifactNtOpen( + root windows.Handle, + name string, + access uint32, + disposition uint32, + options uint32, + attributes uint32, +) (windows.Handle, error) { + objectName, err := windows.NewNTUnicodeString(name) + if err != nil { + return windows.InvalidHandle, err + } + oa := &windows.OBJECT_ATTRIBUTES{ + Length: uint32(unsafe.Sizeof(windows.OBJECT_ATTRIBUTES{})), + RootDirectory: root, + ObjectName: objectName, + Attributes: windows.OBJ_CASE_INSENSITIVE, + } + var handle windows.Handle + var iosb windows.IO_STATUS_BLOCK + var allocationSize int64 + err = windows.NtCreateFile( + &handle, + access, + oa, + &iosb, + &allocationSize, + attributes, + windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE, + disposition, + options, + 0, + 0, + ) + return handle, err +} + +func duplicateArtifactHandle(source windows.Handle) (windows.Handle, error) { + var target windows.Handle + err := windows.DuplicateHandle( + windows.CurrentProcess(), + source, + windows.CurrentProcess(), + &target, + 0, + false, + windows.DUPLICATE_SAME_ACCESS, + ) + return target, err +} + +func rejectArtifactReparsePoint(handle windows.Handle) error { + var info windows.ByHandleFileInformation + if err := windows.GetFileInformationByHandle(handle, &info); err != nil { + return err + } + if info.FileAttributes&windows.FILE_ATTRIBUTE_REPARSE_POINT != 0 { + return errArtifactReparsePoint + } + return nil +} + +func artifactNTAbsolutePath(absolute string) string { + if strings.HasPrefix(absolute, `\\`) { + return `\??\UNC\` + strings.TrimPrefix(absolute, `\\`) + } + return `\??\` + absolute +} + +func artifactPathComponentsWindows(relativePath string) ([]string, error) { + if relativePath == "." { + return nil, nil + } + clean, err := validateArtifactRelativePath(relativePath) + if err != nil { + return nil, err + } + return strings.Split(clean, "/"), nil +} + +func translateArtifactWindowsError(err error) error { + if err == nil { + return nil + } + return err +} + +func isArtifactWindowsNotExist(err error) bool { + return errors.Is(err, windows.ERROR_FILE_NOT_FOUND) || + errors.Is(err, windows.ERROR_PATH_NOT_FOUND) || + errors.Is(err, windows.STATUS_OBJECT_NAME_NOT_FOUND) || + errors.Is(err, windows.STATUS_OBJECT_PATH_NOT_FOUND) +} + +func isArtifactAlreadyExists(err error) bool { + return errors.Is(err, windows.ERROR_ALREADY_EXISTS) || + errors.Is(err, windows.ERROR_FILE_EXISTS) || + errors.Is(err, windows.STATUS_OBJECT_NAME_COLLISION) || + errors.Is(err, ErrArtifactPublishConflict) +} + +func isArtifactNotExist(err error) bool { + return isArtifactWindowsNotExist(err) +} diff --git a/internal/tools/delegation_child_run_mode_test.go b/internal/tools/delegation_child_run_mode_test.go new file mode 100644 index 00000000..495d6a2a --- /dev/null +++ b/internal/tools/delegation_child_run_mode_test.go @@ -0,0 +1,80 @@ +package tools + +import ( + "context" + "strings" + "testing" + + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +func delegationArtifactTestContext() context.Context { + ctx := store.WithTenantID(context.Background(), store.MasterTenantID) + ctx = WithDelegationID(ctx, "11111111-1111-1111-1111-111111111111") + return WithDelegationArtifactInputs(ctx, "/runtime/delegations/input") +} + +func TestDelegationArtifactRunRejectsAsyncChildModes(t *testing.T) { + ctx := delegationArtifactTestContext() + for _, mode := range []string{"async", "spawn", "background"} { + if err := validateDelegationChildRunMode(ctx, "delegate", mode); err == nil { + t.Fatalf("mode %q was accepted", mode) + } + } + if err := validateDelegationChildRunMode(ctx, "delegate", "sync"); err != nil { + t.Fatalf("sync mode rejected: %v", err) + } + if err := validateDelegationChildRunMode(context.Background(), "delegate", "async"); err != nil { + t.Fatalf("normal async run rejected: %v", err) + } +} + +func TestSpawnToolRejectsDefaultAsyncInsideDelegationArtifactRun(t *testing.T) { + tool := NewSpawnTool(nil, "parent", 0) + result := tool.Execute(delegationArtifactTestContext(), map[string]any{ + "task": "must not escape the delegation lifetime", + }) + if result == nil || !strings.Contains(result.ForLLM, "mode=\"sync\"") { + t.Fatalf("unexpected result: %#v", result) + } +} + +func TestDelegateToolRejectsAsyncInsideDelegationArtifactRunBeforeLookup(t *testing.T) { + tool := &DelegateTool{} + result := tool.Execute(delegationArtifactTestContext(), map[string]any{ + "agent_key": "child", + "task": "must not escape the delegation lifetime", + "mode": "async", + }) + if result == nil || !strings.Contains(result.ForLLM, "mode=\"sync\"") { + t.Fatalf("unexpected result: %#v", result) + } +} + +func TestDelegateToolSchemaKeepsBoundedInputsOptional(t *testing.T) { + schema := (&DelegateTool{}).Parameters() + properties, ok := schema["properties"].(map[string]any) + if !ok { + t.Fatalf("properties = %#v", schema["properties"]) + } + inputs, ok := properties["inputs"].(map[string]any) + if !ok { + t.Fatalf("inputs schema = %#v", properties["inputs"]) + } + if got := inputs["type"]; got != "array" { + t.Fatalf("inputs type = %#v, want array", got) + } + if got := inputs["maxItems"]; got != DelegationArtifactMaxFiles { + t.Fatalf("inputs maxItems = %#v, want %d", got, DelegationArtifactMaxFiles) + } + if required, ok := schema["required"].([]string); ok { + for _, name := range required { + if name == "inputs" { + t.Fatal("inputs unexpectedly became required") + } + } + } + if _, ok := properties["delegation_id"]; !ok { + t.Fatal("schema omitted durable delegation result lookup") + } +} diff --git a/internal/tools/edit.go b/internal/tools/edit.go index 3d1b8646..3addabbf 100644 --- a/internal/tools/edit.go +++ b/internal/tools/edit.go @@ -106,6 +106,9 @@ func (t *EditTool) Execute(ctx context.Context, args map[string]any) *Result { if oldStr == newStr { return ErrorResult("old_string and new_string are identical") } + if err := rejectDelegationInputMutation(ctx, path); err != nil { + return ErrorResult(err.Error()) + } // Group write permission check if t.permStore != nil { @@ -115,7 +118,7 @@ func (t *EditTool) Execute(ctx context.Context, args map[string]any) *Result { } // Virtual FS: context files - if t.contextFileIntc != nil { + if !IsDelegationArtifactRun(ctx) && t.contextFileIntc != nil { if content, handled, err := t.contextFileIntc.ReadFile(ctx, path); handled { if err != nil { return ErrorResult(fmt.Sprintf("failed to read context file: %v", err)) @@ -135,7 +138,7 @@ func (t *EditTool) Execute(ctx context.Context, args map[string]any) *Result { } // Virtual FS: memory files - if t.memIntc != nil { + if !IsDelegationArtifactRun(ctx) && t.memIntc != nil { if content, handled, err := t.memIntc.ReadFile(ctx, path); handled { if err != nil { return ErrorResult(fmt.Sprintf("failed to read memory file: %v", err)) @@ -178,7 +181,6 @@ func (t *EditTool) Execute(ctx context.Context, args map[string]any) *Result { if err := checkDeniedPath(resolved, t.workspace, t.deniedPrefixes); err != nil { return ErrorResult(err.Error()) } - data, err := os.ReadFile(resolved) if err != nil { return ErrorResult(fmt.Sprintf("failed to read file: %v", err)) @@ -211,12 +213,12 @@ func (t *EditTool) executeInSandbox(ctx context.Context, path, oldStr, newStr st if err != nil { return ErrorResult(err.Error()) } - containerCwd, cwdErr := sandboxCwdForHostPath(mountWorkspace, mountWorkspace, sandbox.DefaultContainerWorkdir) + containerCwd, cwdErr := sandboxCwdForHostPath(mountWorkspace, mountWorkspace, sandboxContainerWorkdir(ctx)) if cwdErr != nil { return ErrorResult(fmt.Sprintf("sandbox path mapping: %v", cwdErr)) } - sb, err := t.sandboxMgr.Get(ctx, sandboxKey, mountWorkspace, SandboxConfigFromCtx(ctx)) + sb, err := acquireToolSandbox(ctx, t.sandboxMgr, sandboxKey, mountWorkspace) if err != nil { return ErrorResult(fmt.Sprintf("sandbox error: %v", err)) } diff --git a/internal/tools/filesystem.go b/internal/tools/filesystem.go index e257f600..3310c53d 100644 --- a/internal/tools/filesystem.go +++ b/internal/tools/filesystem.go @@ -117,8 +117,10 @@ func (t *ReadFileTool) Execute(ctx context.Context, args map[string]any) *Result } } - // Virtual FS: route context files to DB - if t.contextFileIntc != nil { + // Agent Link artifact runs are a physical exchange boundary. Virtual + // context files belong to the delegate's persistent identity and must not + // masquerade as staged inputs. + if !IsDelegationArtifactRun(ctx) && t.contextFileIntc != nil { if content, handled, err := t.contextFileIntc.ReadFile(ctx, path); handled { if err != nil { return ErrorResult(fmt.Sprintf("failed to read context file: %v", err)) @@ -132,13 +134,16 @@ func (t *ReadFileTool) Execute(ctx context.Context, args map[string]any) *Result // Virtual system files: TEAM.md, DELEGATION.md, AVAILABILITY.md are injected // into the system prompt and don't exist on disk. Return a helpful hint. - baseName := filepath.Base(path) - if hint, ok := virtualSystemFiles[baseName]; ok { - return SilentResult(hint) + if !IsDelegationArtifactRun(ctx) { + baseName := filepath.Base(path) + if hint, ok := virtualSystemFiles[baseName]; ok { + return SilentResult(hint) + } } - // Virtual FS: route memory files to DB - if t.memIntc != nil { + // Delegated runs use physical exchange files only. Context and memory + // virtual-file routing must not cross the Agent Link boundary. + if !IsDelegationArtifactRun(ctx) && t.memIntc != nil { if content, handled, err := t.memIntc.ReadFile(ctx, path); handled { if err != nil { return ErrorResult(fmt.Sprintf("failed to read memory file: %v", err)) @@ -150,12 +155,31 @@ func (t *ReadFileTool) Execute(ctx context.Context, args map[string]any) *Result } } - // Sandbox routing (sandboxKey from ctx — thread-safe) + // Sandboxed delegation inputs are addressed by their logical alias. The + // acquisition helper mounts only this exchange at /workspace/inputs:ro. sandboxKey := ToolSandboxKeyFromCtx(ctx) if t.sandboxMgr != nil && sandboxKey != "" { return t.executeInSandbox(ctx, path, sandboxKey, args) } + if resolved, handled, err := resolveDelegationInputPath(ctx, path); handled { + if err != nil { + return ErrorResult("cannot access delegation input") + } + if err := ValidateRegularFileForRead(resolved); err != nil { + return ErrorResult("delegation input is not a safe regular file") + } + if isBinaryFileExt(resolved) { + ext := strings.ToLower(filepath.Ext(resolved)) + return ErrorResult(fmt.Sprintf("cannot read binary file (%s). Use the appropriate tool: read_image for images, read_document for documents, read_audio for audio, read_video for video.", ext)) + } + data, err := os.ReadFile(resolved) + if err != nil { + return ErrorResult("failed to read delegation input") + } + return t.paginateOutput(string(data), args) + } + // Host execution — use per-user workspace from context if available workspace := ToolWorkspaceFromCtx(ctx) if workspace == "" { @@ -169,6 +193,9 @@ func (t *ReadFileTool) Execute(ctx context.Context, args map[string]any) *Result if err := checkDeniedPath(resolved, t.workspace, t.deniedPrefixes); err != nil { return ErrorResult(err.Error()) } + if err := ValidateRegularFileForRead(resolved); err != nil { + return ErrorResult(fmt.Sprintf("cannot read path: %v", err)) + } // Block binary files — reading them wastes context with garbled data. if isBinaryFileExt(resolved) { @@ -200,7 +227,7 @@ func (t *ReadFileTool) executeInSandbox(ctx context.Context, path, sandboxKey st if err != nil { return ErrorResult(err.Error()) } - containerCwd, cwdErr := sandboxCwdForHostPath(mountWorkspace, mountWorkspace, sandbox.DefaultContainerWorkdir) + containerCwd, cwdErr := sandboxCwdForHostPath(mountWorkspace, mountWorkspace, sandboxContainerWorkdir(ctx)) if cwdErr != nil { return ErrorResult(fmt.Sprintf("sandbox path mapping: %v", cwdErr)) } @@ -219,7 +246,7 @@ func (t *ReadFileTool) executeInSandbox(ctx context.Context, path, sandboxKey st } func (t *ReadFileTool) getFsBridge(ctx context.Context, sandboxKey, mountWorkspace, containerCwd string) (*sandbox.FsBridge, error) { - sb, err := t.sandboxMgr.Get(ctx, sandboxKey, mountWorkspace, SandboxConfigFromCtx(ctx)) + sb, err := acquireToolSandbox(ctx, t.sandboxMgr, sandboxKey, mountWorkspace) if err != nil { return nil, err } @@ -332,9 +359,8 @@ func allowedWriteWithTeamWorkspace(ctx context.Context, base []string) []string return buildAllowedPrefixes(ctx, base, false) } -// buildAllowedPrefixes merges base + tenant paths + team workspace, optionally -// including team root. Extracted to share the slice-building logic between read -// and write variants without duplication. +// buildAllowedPrefixes merges base + tenant paths + team workspace and, +// for read operations, the explicit Agent Team root. func buildAllowedPrefixes(ctx context.Context, base []string, includeTeamRoot bool) []string { tenantPaths := TenantAllowedPathsFromCtx(ctx) teamWs := ToolTeamWorkspaceFromCtx(ctx) @@ -342,7 +368,6 @@ func buildAllowedPrefixes(ctx context.Context, base []string, includeTeamRoot bo if includeTeamRoot { teamRoot = ToolTeamRootFromCtx(ctx) } - if len(tenantPaths) == 0 && teamWs == "" && teamRoot == "" { return base } @@ -393,6 +418,16 @@ func resolvePathWithAllowed(path, workspace string, restrict bool, allowedPrefix prefixReal = absPrefix } if isPathInside(real, prefixReal) { + // Preserve the same filesystem hardening as resolvePath. Allowed + // prefixes widen the containment boundary only; they must not + // weaken symlink or hardlink protections. + if hasMutableSymlinkParent(real) { + slog.Warn("security.mutable_symlink_parent", "path", path, "resolved", real) + return "", fmt.Errorf("access denied: path contains mutable symlink component") + } + if err := checkHardlink(real); err != nil { + return "", err + } slog.Debug("read_file: allowed by prefix", "path", real, "prefix", prefixReal) return real, nil } @@ -401,6 +436,20 @@ func resolvePathWithAllowed(path, workspace string, restrict bool, allowedPrefix return "", err } +// ValidateRegularFileForRead enforces the final leaf-file boundary used by +// outbound media paths. It rejects missing paths, directories, devices, +// sockets, FIFOs, symlinks, and hardlinked regular files. +func ValidateRegularFileForRead(path string) error { + info, err := os.Lstat(path) + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return fmt.Errorf("path is not a regular file") + } + return checkHardlink(path) +} + // checkDeniedPath returns an error if the resolved path falls under any denied prefix. // Denied prefixes are relative to the workspace (e.g. ".goclaw" denies workspace/.goclaw/). // The resolved path should already be canonical (from resolvePath with restrict=true). diff --git a/internal/tools/filesystem_list.go b/internal/tools/filesystem_list.go index 52d1cdcd..2095d940 100644 --- a/internal/tools/filesystem_list.go +++ b/internal/tools/filesystem_list.go @@ -72,7 +72,7 @@ func (t *ListFilesTool) Execute(ctx context.Context, args map[string]any) *Resul } // Virtual FS: route memory directory listing to DB - if t.memIntc != nil { + if !IsDelegationArtifactRun(ctx) && t.memIntc != nil { if listing, handled, err := t.memIntc.ListFiles(ctx, path); handled { if err != nil { return ErrorResult(fmt.Sprintf("failed to list memory files: %v", err)) @@ -90,6 +90,13 @@ func (t *ListFilesTool) Execute(ctx context.Context, args map[string]any) *Resul return t.executeInSandbox(ctx, path, sandboxKey) } + if resolved, handled, err := resolveDelegationInputPath(ctx, path); handled { + if err != nil { + return ErrorResult("cannot access delegation input") + } + return t.executeDelegationHostList(ctx, resolved, path) + } + // Host execution — use per-user workspace from context if available workspace := ToolWorkspaceFromCtx(ctx) if workspace == "" { @@ -104,12 +111,24 @@ func (t *ListFilesTool) Execute(ctx context.Context, args map[string]any) *Resul return ErrorResult(err.Error()) } + return t.executeHostList(ctx, resolved, path) +} + +func (t *ListFilesTool) executeDelegationHostList(ctx context.Context, resolved, displayPath string) *Result { + result := t.executeHostList(ctx, resolved, displayPath) + if result.IsError { + return ErrorResult("failed to list delegation input directory") + } + return result +} + +func (t *ListFilesTool) executeHostList(ctx context.Context, resolved, displayPath string) *Result { entries, err := os.ReadDir(resolved) if err != nil { if os.IsNotExist(err) { - msg := fmt.Sprintf("Directory does not exist: %s", path) + msg := fmt.Sprintf("Directory does not exist: %s", displayPath) if teamWs := ToolTeamWorkspaceFromCtx(ctx); teamWs != "" && !strings.HasPrefix(resolved, teamWs) { - msg += fmt.Sprintf("\nHint: try the team workspace path: list_files(path=\"%s/%s\")", teamWs, path) + msg += fmt.Sprintf("\nHint: try the team workspace path: list_files(path=\"%s/%s\")", teamWs, displayPath) } return SilentResult(msg) } @@ -144,7 +163,7 @@ func (t *ListFilesTool) executeInSandbox(ctx context.Context, path, sandboxKey s if err != nil { return ErrorResult(err.Error()) } - containerCwd, cwdErr := sandboxCwdForHostPath(mountWorkspace, mountWorkspace, sandbox.DefaultContainerWorkdir) + containerCwd, cwdErr := sandboxCwdForHostPath(mountWorkspace, mountWorkspace, sandboxContainerWorkdir(ctx)) if cwdErr != nil { return ErrorResult(fmt.Sprintf("sandbox path mapping: %v", cwdErr)) } @@ -163,7 +182,7 @@ func (t *ListFilesTool) executeInSandbox(ctx context.Context, path, sandboxKey s } func (t *ListFilesTool) getFsBridge(ctx context.Context, sandboxKey, mountWorkspace, containerCwd string) (*sandbox.FsBridge, error) { - sb, err := t.sandboxMgr.Get(ctx, sandboxKey, mountWorkspace, SandboxConfigFromCtx(ctx)) + sb, err := acquireToolSandbox(ctx, t.sandboxMgr, sandboxKey, mountWorkspace) if err != nil { return nil, err } diff --git a/internal/tools/filesystem_write.go b/internal/tools/filesystem_write.go index ce7ed448..4fb119a1 100644 --- a/internal/tools/filesystem_write.go +++ b/internal/tools/filesystem_write.go @@ -114,6 +114,12 @@ func (t *WriteFileTool) Execute(ctx context.Context, args map[string]any) *Resul if path == "" { return ErrorResult("path is required") } + if err := rejectDelegationInputMutation(ctx, path); err != nil { + return ErrorResult(err.Error()) + } + if IsDelegationArtifactRun(ctx) { + deliver = false + } // Group write permission check if t.permStore != nil { @@ -123,7 +129,7 @@ func (t *WriteFileTool) Execute(ctx context.Context, args map[string]any) *Resul } // Virtual FS: route context files to DB - if t.contextFileIntc != nil { + if !IsDelegationArtifactRun(ctx) && t.contextFileIntc != nil { if handled, err := t.contextFileIntc.WriteFile(ctx, path, content); handled { if err != nil { return ErrorResult(fmt.Sprintf("failed to write context file: %v", err)) @@ -133,7 +139,7 @@ func (t *WriteFileTool) Execute(ctx context.Context, args map[string]any) *Resul } // Virtual FS: route memory files to DB - if t.memIntc != nil { + if !IsDelegationArtifactRun(ctx) && t.memIntc != nil { if mwr, err := t.memIntc.WriteFile(ctx, path, content, appendMode); mwr.Handled { if err != nil { return ErrorResult(fmt.Sprintf("failed to write memory file: %v", err)) @@ -244,7 +250,7 @@ func (t *WriteFileTool) executeInSandbox(ctx context.Context, path, content, san if err != nil { return ErrorResult(err.Error()) } - containerCwd, cwdErr := sandboxCwdForHostPath(mountWorkspace, mountWorkspace, sandbox.DefaultContainerWorkdir) + containerCwd, cwdErr := sandboxCwdForHostPath(mountWorkspace, mountWorkspace, sandboxContainerWorkdir(ctx)) if cwdErr != nil { return ErrorResult(fmt.Sprintf("sandbox path mapping: %v", cwdErr)) } @@ -288,7 +294,7 @@ func (t *WriteFileTool) executeInSandbox(ctx context.Context, path, content, san } func (t *WriteFileTool) getFsBridge(ctx context.Context, sandboxKey, mountWorkspace, containerCwd string) (*sandbox.FsBridge, error) { - sb, err := t.sandboxMgr.Get(ctx, sandboxKey, mountWorkspace, SandboxConfigFromCtx(ctx)) + sb, err := acquireToolSandbox(ctx, t.sandboxMgr, sandboxKey, mountWorkspace) if err != nil { return nil, err } diff --git a/internal/tools/message.go b/internal/tools/message.go index f4709dc3..6687fcd9 100644 --- a/internal/tools/message.go +++ b/internal/tools/message.go @@ -118,6 +118,9 @@ func (t *MessageTool) Execute(ctx context.Context, args map[string]any) *Result if message == "" { return ErrorResult("message is required") } + if IsDelegationArtifactRun(ctx) && strings.Contains(message, "MEDIA:") { + return ErrorResult("delegation files are published only after the delegated run completes") + } channel := argString(args, "channel") if channel == "" { @@ -470,8 +473,8 @@ func (t *MessageTool) validateChannelTenant(ctx context.Context, channel, target // sendMedia sends a file as a media attachment via the outbound message bus. func (t *MessageTool) sendMedia(ctx context.Context, channel, target, filePath string) *Result { - if _, err := os.Stat(filePath); err != nil { - return ErrorResult(fmt.Sprintf("file not found: %s", filePath)) + if err := ValidateRegularFileForRead(filePath); err != nil { + return ErrorResult(fmt.Sprintf("media path is not a safe regular file: %v", err)) } if t.msgBus == nil { return ErrorResult("media sending requires message bus") @@ -530,6 +533,9 @@ func (t *MessageTool) extractEmbeddedMedia(ctx context.Context, message string) // Extract each MEDIA: path and resolve via security-checked path resolution. for _, raw := range matches { if resolved, ok := t.resolveMediaPath(ctx, raw); ok { + if err := ValidateRegularFileForRead(resolved); err != nil { + continue + } media = append(media, bus.MediaAttachment{ URL: resolved, ContentType: mimeFromPath(resolved), @@ -622,11 +628,10 @@ func isGroupContext(ctx context.Context) bool { // resolveMediaPath extracts and validates a file path from a "MEDIA:path" string. // Uses the same workspace-aware path resolution as other filesystem tools. -// Multi-tenant isolation forces MEDIA: paths through restricted resolution -// first, with one explicit fallback for generated media artifacts under /tmp/. +// Multi-tenant isolation forces MEDIA: paths through restricted resolution. // In practice MEDIA: paths may resolve to: // - files inside the agent workspace -// - absolute paths under /tmp/ for generated media artifacts +// - files inside an explicitly authorized team/collaboration read root // // Relative paths are resolved against the agent's workspace. func (t *MessageTool) resolveMediaPath(ctx context.Context, s string) (string, bool) { @@ -645,17 +650,10 @@ func (t *MessageTool) resolveMediaPath(ctx context.Context, s string) (string, b } restrict := effectiveRestrict(ctx, t.restrict) - // resolvePath handles relative→absolute, symlink, hardlink, boundary checks. - resolved, err := resolvePath(raw, workspace, restrict) + // Use the read-only prefix set so orchestrators can publish media produced + // by linked agents or team members without granting mutation access. + resolved, err := resolvePathWithAllowed(raw, workspace, restrict, allowedWithTeamWorkspace(ctx, nil)) if err != nil { - // When restricted, also allow /tmp/ paths (used by create_image, create_audio, etc.) - // But reject paths that are siblings of the workspace — these are likely traversal - // attacks where workspace/../X resolves inside /tmp/ because workspace itself is in /tmp/. - cleaned := filepath.Clean(raw) - wsParent := filepath.Dir(filepath.Clean(workspace)) - if restrict && isInTempDir(cleaned) && !isPathInside(cleaned, wsParent) { - return cleaned, true - } return "", false } diff --git a/internal/tools/message_test.go b/internal/tools/message_test.go index 18c5c6f6..0289c0d5 100644 --- a/internal/tools/message_test.go +++ b/internal/tools/message_test.go @@ -56,9 +56,9 @@ func TestResolveMediaPath(t *testing.T) { want string wantOK bool }{ - // /tmp/ always allowed - {"valid temp file", "MEDIA:" + filepath.Join(tmpDir, "test.png"), filepath.Join(tmpDir, "test.png"), true}, - {"valid nested temp", "MEDIA:" + filepath.Join(tmpDir, "sub", "file.txt"), filepath.Join(tmpDir, "sub", "file.txt"), true}, + // Unscoped temp files are not tenant/run-owned. + {"unscoped temp file", "MEDIA:" + filepath.Join(tmpDir, "test.png"), "", false}, + {"unscoped nested temp", "MEDIA:" + filepath.Join(tmpDir, "sub", "file.txt"), "", false}, // Workspace files allowed {"workspace absolute", "MEDIA:" + testFileCanonical, testFileCanonical, true}, @@ -104,8 +104,8 @@ func TestResolveMediaPath(t *testing.T) { {"absolute outside workspace", "MEDIA:" + outsidePath(workspaceCanonical, "etc/hostname"), false}, // Workspace-relative → allowed {"workspace relative", "MEDIA:docs/report.pdf", true}, - // /tmp/ → allowed (temp dir exception in restricted mode) - {"temp file", "MEDIA:" + filepath.Join(tmpDir, "test.png"), true}, + // /tmp/ is not an implicit cross-tenant read root. + {"temp file", "MEDIA:" + filepath.Join(tmpDir, "test.png"), false}, } for _, tt := range tests { @@ -131,6 +131,29 @@ func TestResolveMediaPath(t *testing.T) { t.Errorf("got %q, want %q", got, testFileCanonical) } }) + +} + +func TestMessageMediaRejectsNonRegularFiles(t *testing.T) { + workspace := t.TempDir() + tool := NewMessageTool(workspace, true) + tool.SetMessageBus(bus.New()) + + if result := tool.sendMedia(context.Background(), "telegram", "chat-1", workspace); !result.IsError { + t.Fatal("sendMedia allowed a directory") + } + + original := filepath.Join(workspace, "original.png") + if err := os.WriteFile(original, []byte("image"), 0o644); err != nil { + t.Fatal(err) + } + hardlink := filepath.Join(workspace, "hardlink.png") + if err := os.Link(original, hardlink); err != nil { + t.Skipf("hardlinks not supported: %v", err) + } + if result := tool.sendMedia(context.Background(), "telegram", "chat-1", hardlink); !result.IsError { + t.Fatal("sendMedia allowed a hardlinked file") + } } func TestIsInTempDir(t *testing.T) { @@ -157,8 +180,6 @@ func TestIsInTempDir(t *testing.T) { } func TestExtractEmbeddedMedia(t *testing.T) { - tmpDir := os.TempDir() - workspace := t.TempDir() workspaceCanonical, _ := filepath.EvalSymlinks(workspace) @@ -215,7 +236,10 @@ func TestExtractEmbeddedMedia(t *testing.T) { }) t.Run("multiple MEDIA: on same line", func(t *testing.T) { - img := filepath.Join(tmpDir, "photo.png") + img := filepath.Join(workspaceCanonical, "photo.png") + if err := os.WriteFile(img, []byte("image"), 0o644); err != nil { + t.Fatal(err) + } msg := "MEDIA:" + reportCanonical + " MEDIA:" + img cleaned, media := tool.extractEmbeddedMedia(ctx, msg) @@ -227,6 +251,17 @@ func TestExtractEmbeddedMedia(t *testing.T) { } }) + t.Run("non-regular MEDIA path is stripped without attachment", func(t *testing.T) { + msg := "Directory:\nMEDIA:" + docsDir + "\nDone" + cleaned, media := tool.extractEmbeddedMedia(ctx, msg) + if cleaned != "Directory:\nDone" { + t.Fatalf("unexpected cleaned text: %q", cleaned) + } + if len(media) != 0 { + t.Fatalf("non-regular path produced attachments: %+v", media) + } + }) + t.Run("MEDIA: path outside workspace is stripped but no attachment", func(t *testing.T) { msg := "File:\nMEDIA:" + outsidePath(workspaceCanonical, "etc/passwd") + "\nDone" cleaned, media := tool.extractEmbeddedMedia(ctx, msg) @@ -252,7 +287,11 @@ func TestExtractEmbeddedMedia(t *testing.T) { }) t.Run("audio_as_voice tag stripped", func(t *testing.T) { - msg := "[[audio_as_voice]]\nMEDIA:" + filepath.Join(tmpDir, "voice.ogg") + "\nExtra text" + voice := filepath.Join(workspaceCanonical, "voice.ogg") + if err := os.WriteFile(voice, []byte("audio"), 0o644); err != nil { + t.Fatal(err) + } + msg := "[[audio_as_voice]]\nMEDIA:" + voice + "\nExtra text" cleaned, media := tool.extractEmbeddedMedia(ctx, msg) if cleaned != "Extra text" { @@ -264,7 +303,10 @@ func TestExtractEmbeddedMedia(t *testing.T) { }) t.Run("multiple MEDIA: paths", func(t *testing.T) { - img := filepath.Join(tmpDir, "photo.png") + img := filepath.Join(workspaceCanonical, "photo-2.png") + if err := os.WriteFile(img, []byte("image"), 0o644); err != nil { + t.Fatal(err) + } msg := "Files:\nMEDIA:" + reportCanonical + "\nMEDIA:" + img + "\nEnjoy!" cleaned, media := tool.extractEmbeddedMedia(ctx, msg) @@ -317,8 +359,8 @@ func TestValidateChannelTenant(t *testing.T) { // Wire a mock checker. channels := map[string]uuid.UUID{ - "telegram": tenantA, - "tenant-b-tg": tenantB, + "telegram": tenantA, + "tenant-b-tg": tenantB, } tool.SetChannelTenantChecker(func(name string) (uuid.UUID, bool) { tid, ok := channels[name] diff --git a/internal/tools/read_audio.go b/internal/tools/read_audio.go index a329dd26..81230316 100644 --- a/internal/tools/read_audio.go +++ b/internal/tools/read_audio.go @@ -5,6 +5,7 @@ import ( "fmt" "log/slog" "os" + "path/filepath" "strings" "github.com/nextlevelbuilder/goclaw/internal/providers" @@ -62,7 +63,7 @@ func (t *ReadAudioTool) Name() string { return "read_audio" } func (t *ReadAudioTool) Description() string { return "Analyze audio files (speech, music, sounds) attached to the conversation. " + "Use when you see tags and need to transcribe, summarize, or analyze audio content. " + - "Specify what you want to extract or analyze." + "A workspace-relative path such as inputs/recording.mp3 may also be provided. Specify what you want to extract or analyze." } func (t *ReadAudioTool) Parameters() map[string]any { @@ -77,6 +78,10 @@ func (t *ReadAudioTool) Parameters() map[string]any { "type": "string", "description": "Optional: specific media_id from tag. If omitted, uses most recent audio.", }, + "path": map[string]any{ + "type": "string", + "description": "Optional workspace-relative audio path. Delegated inputs use inputs/.", + }, }, "required": []string{"prompt"}, } @@ -88,16 +93,30 @@ func (t *ReadAudioTool) Execute(ctx context.Context, args map[string]any) *Resul prompt = "Analyze this audio and describe its contents." } mediaID, _ := args["media_id"].(string) + audioArg, _ := args["path"].(string) + if mediaID != "" && audioArg != "" { + return ErrorResult("Both 'media_id' and 'path' parameters cannot be specified. Choose only one.") + } - audioPath, audioMime, err := t.resolveAudioFile(ctx, mediaID) + var audioPath, audioMime string + var err error + if audioArg != "" { + audioPath, err = resolveStructuredMediaPath(ctx, audioArg, "audio") + audioMime = mimeFromAudioExt(filepath.Ext(audioPath)) + } else { + audioPath, audioMime, err = t.resolveAudioFile(ctx, mediaID) + } if err != nil { return ErrorResult(err.Error()) } - slog.Info("read_audio: resolved file", "path", audioPath, "mime", audioMime, "media_id", mediaID) + slog.Info("read_audio: resolved file", "mime", audioMime, "media_id", mediaID, "logical_path", audioArg) data, err := os.ReadFile(audioPath) if err != nil { + if audioArg != "" && IsDelegationArtifactRun(ctx) { + return ErrorResult("Failed to read delegation audio input") + } return ErrorResult(fmt.Sprintf("Failed to read audio file: %v", err)) } slog.Info("read_audio: file loaded", "size_bytes", len(data)) diff --git a/internal/tools/read_audio_resolve.go b/internal/tools/read_audio_resolve.go index 33ce7eda..35c9c8b3 100644 --- a/internal/tools/read_audio_resolve.go +++ b/internal/tools/read_audio_resolve.go @@ -14,22 +14,11 @@ import ( // resolveAudioFile finds the audio file path from context MediaRefs. func (t *ReadAudioTool) resolveAudioFile(ctx context.Context, mediaID string) (path, mime string, err error) { - if t.mediaLoader == nil { - return "", "", fmt.Errorf("no media storage configured — cannot access audio files") - } - refs := MediaAudioRefsFromCtx(ctx) if len(refs) == 0 { return "", "", fmt.Errorf("no audio files available in this conversation. The user may not have sent an audio file.") } - // Sanitize media_id: LLM may pass the literal tag string (e.g. "") - // instead of a UUID. Treat tag-like values as empty to fall back to most recent. - if strings.Contains(mediaID, "<") || strings.Contains(mediaID, "media:") { - slog.Debug("read_audio: sanitizing tag-like media_id", "raw", mediaID) - mediaID = "" - } - var ref *providers.MediaRef if mediaID != "" { for i := range refs { @@ -39,10 +28,7 @@ func (t *ReadAudioTool) resolveAudioFile(ctx context.Context, mediaID string) (p } } if ref == nil { - // Fallback to most recent audio instead of hard error, - // since LLM may generate invalid IDs. - slog.Warn("read_audio: media_id not found, falling back to most recent", "media_id", mediaID) - ref = &refs[len(refs)-1] + return "", "", fmt.Errorf("audio media_id %q not found in this conversation", mediaID) } } else { ref = &refs[len(refs)-1] @@ -50,6 +36,7 @@ func (t *ReadAudioTool) resolveAudioFile(ctx context.Context, mediaID string) (p // Prefer persisted workspace path; fall back to legacy .media/ lookup. p := ref.Path + loadedLegacy := false if p == "" { var err error if t.mediaLoader == nil { @@ -59,6 +46,16 @@ func (t *ReadAudioTool) resolveAudioFile(ctx context.Context, mediaID string) (p if err != nil { return "", "", fmt.Errorf("audio file not found: %v", err) } + loadedLegacy = true + } + + if loadedLegacy { + p, err = resolveLoadedMediaRefPath(ctx, t.mediaLoader, p, "audio") + } else { + p, err = resolveStructuredMediaRefPath(ctx, p, "audio") + } + if err != nil { + return "", "", err } mime = ref.MimeType diff --git a/internal/tools/read_audio_resolve_test.go b/internal/tools/read_audio_resolve_test.go index df671dba..a2f06920 100644 --- a/internal/tools/read_audio_resolve_test.go +++ b/internal/tools/read_audio_resolve_test.go @@ -2,12 +2,29 @@ package tools import ( "context" + "os" + "path/filepath" "strings" "testing" + "github.com/google/uuid" + "github.com/nextlevelbuilder/goclaw/internal/providers" ) +type testMediaPathLoader struct { + path string + root string +} + +func (l testMediaPathLoader) LoadPath(string) (string, error) { + return l.path, nil +} + +func (l testMediaPathLoader) MediaRootPath() string { + return l.root +} + type readAudioUnsupportedProvider struct { name string chatCalls int @@ -122,3 +139,202 @@ func TestReadAudioCallProvider_UnsupportedProviderDoesNotSendAudioAsImage(t *tes t.Fatalf("unsupported audio route called chat fallback: calls=%d images=%d", fake.chatCalls, fake.images) } } + +func TestResolveAudioFileRequiresExactMediaID(t *testing.T) { + workspace := t.TempDir() + audioPath := filepath.Join(workspace, ".uploads", "recording.mp3") + if err := os.MkdirAll(filepath.Dir(audioPath), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(audioPath, []byte("audio"), 0o644); err != nil { + t.Fatal(err) + } + ref := providers.MediaRef{ + ID: uuid.NewString(), + Kind: "audio", + Path: audioPath, + MimeType: "audio/mpeg", + } + tool := NewReadAudioTool(nil, nil) + ctx := WithToolWorkspace(context.Background(), workspace) + ctx = WithMediaAudioRefs(ctx, []providers.MediaRef{ref}) + + if gotPath, _, err := tool.resolveAudioFile(ctx, "recording.mp3"); err == nil { + t.Fatalf("non-ID value resolved to %q, want exact media_id error", gotPath) + } + gotPath, gotMime, err := tool.resolveAudioFile(ctx, ref.ID) + if err != nil { + t.Fatalf("exact media_id returned error: %v", err) + } + wantPath, err := filepath.EvalSymlinks(audioPath) + if err != nil { + t.Fatal(err) + } + if gotPath != wantPath || gotMime != "audio/mpeg" { + t.Fatalf("resolved (%q, %q), want (%q, audio/mpeg)", gotPath, gotMime, wantPath) + } +} + +func TestResolveAudioFileOmittedMediaIDUsesNewestRef(t *testing.T) { + workspace := t.TempDir() + oldPath := filepath.Join(workspace, ".uploads", "old.mp3") + latestPath := filepath.Join(workspace, ".uploads", "latest.mp3") + if err := os.MkdirAll(filepath.Dir(oldPath), 0o755); err != nil { + t.Fatal(err) + } + for _, path := range []string{oldPath, latestPath} { + if err := os.WriteFile(path, []byte("audio"), 0o644); err != nil { + t.Fatal(err) + } + } + refs := []providers.MediaRef{ + {ID: uuid.NewString(), Kind: "audio", Path: oldPath, MimeType: "audio/mpeg"}, + {ID: uuid.NewString(), Kind: "audio", Path: latestPath, MimeType: "audio/mpeg"}, + } + ctx := WithToolWorkspace(context.Background(), workspace) + ctx = WithMediaAudioRefs(ctx, refs) + + gotPath, _, err := NewReadAudioTool(nil, nil).resolveAudioFile(ctx, "") + if err != nil { + t.Fatalf("omitted media_id returned error: %v", err) + } + wantPath, err := filepath.EvalSymlinks(latestPath) + if err != nil { + t.Fatal(err) + } + if gotPath != wantPath { + t.Fatalf("path = %q, want newest %q", gotPath, wantPath) + } +} + +func TestResolveAudioFileRejectsRefPathOutsideWorkspace(t *testing.T) { + workspace := t.TempDir() + outsidePath := filepath.Join(t.TempDir(), "secret.mp3") + if err := os.WriteFile(outsidePath, []byte("secret"), 0o644); err != nil { + t.Fatal(err) + } + ref := providers.MediaRef{ + ID: uuid.NewString(), + Kind: "audio", + Path: outsidePath, + MimeType: "audio/mpeg", + } + ctx := WithToolWorkspace(context.Background(), workspace) + ctx = WithMediaAudioRefs(ctx, []providers.MediaRef{ref}) + + gotPath, _, err := NewReadAudioTool(nil, nil).resolveAudioFile(ctx, ref.ID) + if err == nil { + t.Fatalf("outside ref path resolved to %q, want containment error", gotPath) + } +} + +func TestResolveAudioFileRejectsLegacyLoaderPathOutsideWorkspace(t *testing.T) { + workspace := t.TempDir() + outsidePath := filepath.Join(t.TempDir(), "legacy.mp3") + if err := os.WriteFile(outsidePath, []byte("audio"), 0o644); err != nil { + t.Fatal(err) + } + ref := providers.MediaRef{ + ID: uuid.NewString(), + Kind: "audio", + MimeType: "audio/mpeg", + } + ctx := WithToolWorkspace(context.Background(), workspace) + ctx = WithMediaAudioRefs(ctx, []providers.MediaRef{ref}) + + gotPath, _, err := NewReadAudioTool(nil, testMediaPathLoader{path: outsidePath}).resolveAudioFile(ctx, ref.ID) + if err == nil { + t.Fatalf("outside legacy loader path resolved to %q, want containment error", gotPath) + } +} + +func TestResolveAudioFileAcceptsConfiguredLegacyMediaRoot(t *testing.T) { + workspace := t.TempDir() + mediaRoot := filepath.Join(t.TempDir(), ".media") + audioPath := filepath.Join(mediaRoot, "session-hash", "legacy.mp3") + if err := os.MkdirAll(filepath.Dir(audioPath), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(audioPath, []byte("audio"), 0o644); err != nil { + t.Fatal(err) + } + ref := providers.MediaRef{ + ID: uuid.NewString(), + Kind: "audio", + MimeType: "audio/mpeg", + } + ctx := WithToolWorkspace(context.Background(), workspace) + ctx = WithMediaAudioRefs(ctx, []providers.MediaRef{ref}) + + gotPath, _, err := NewReadAudioTool(nil, testMediaPathLoader{ + path: audioPath, + root: mediaRoot, + }).resolveAudioFile(ctx, ref.ID) + if err != nil { + t.Fatalf("configured legacy media path returned error: %v", err) + } + wantPath, err := filepath.EvalSymlinks(audioPath) + if err != nil { + t.Fatal(err) + } + if gotPath != wantPath { + t.Fatalf("path = %q, want %q", gotPath, wantPath) + } +} + +func TestResolveAudioFileAcceptsLegacyLoaderPathInsideWorkspace(t *testing.T) { + workspace := t.TempDir() + audioPath := filepath.Join(workspace, ".uploads", "legacy.mp3") + if err := os.MkdirAll(filepath.Dir(audioPath), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(audioPath, []byte("audio"), 0o644); err != nil { + t.Fatal(err) + } + ref := providers.MediaRef{ + ID: uuid.NewString(), + Kind: "audio", + MimeType: "audio/mpeg", + } + ctx := WithToolWorkspace(context.Background(), workspace) + ctx = WithMediaAudioRefs(ctx, []providers.MediaRef{ref}) + + gotPath, _, err := NewReadAudioTool(nil, testMediaPathLoader{path: audioPath}).resolveAudioFile(ctx, ref.ID) + if err != nil { + t.Fatalf("inside legacy loader path returned error: %v", err) + } + wantPath, err := filepath.EvalSymlinks(audioPath) + if err != nil { + t.Fatal(err) + } + if gotPath != wantPath { + t.Fatalf("path = %q, want %q", gotPath, wantPath) + } +} + +func TestResolveAudioFileAcceptsDelegationInputRef(t *testing.T) { + ctx, inputs, _ := delegationArtifactToolContext(t) + audioPath := filepath.Join(inputs, "recording.mp3") + if err := os.WriteFile(audioPath, []byte("audio"), 0o440); err != nil { + t.Fatal(err) + } + ref := providers.MediaRef{ + ID: uuid.NewString(), + Kind: "audio", + Path: audioPath, + MimeType: "audio/mpeg", + } + ctx = WithMediaAudioRefs(ctx, []providers.MediaRef{ref}) + + gotPath, _, err := NewReadAudioTool(nil, nil).resolveAudioFile(ctx, ref.ID) + if err != nil { + t.Fatalf("delegation input ref returned error: %v", err) + } + wantPath, err := filepath.EvalSymlinks(audioPath) + if err != nil { + t.Fatal(err) + } + if gotPath != wantPath { + t.Fatalf("path = %q, want staged input %q", gotPath, wantPath) + } +} diff --git a/internal/tools/read_document.go b/internal/tools/read_document.go index cd7844cc..ac8d05f4 100644 --- a/internal/tools/read_document.go +++ b/internal/tools/read_document.go @@ -5,9 +5,12 @@ import ( "fmt" "log/slog" "os" + "path" + "path/filepath" "strings" "github.com/nextlevelbuilder/goclaw/internal/providers" + "github.com/nextlevelbuilder/goclaw/internal/tracing" usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps" ) @@ -123,6 +126,9 @@ func (t *ReadDocumentTool) Execute(ctx context.Context, args map[string]any) *Re } mediaID, _ := args["media_id"].(string) docPathArg, _ := args["path"].(string) + if strings.TrimSpace(mediaID) != "" && strings.TrimSpace(docPathArg) != "" { + return ErrorResult("Specify either media_id or path, not both.") + } // Resolve document file path from MediaRefs in context. docPath, docMime, err := t.resolveDocumentFile(ctx, mediaID, docPathArg) @@ -130,19 +136,24 @@ func (t *ReadDocumentTool) Execute(ctx context.Context, args map[string]any) *Re return ErrorResult(err.Error()) } - slog.Info("read_document: resolved file", "path", docPath, "mime", docMime, "media_id", mediaID) + displayPath := logicalDocumentDisplayPath(ctx, docPath, docPathArg) + slog.Info("read_document: resolved file", "path", displayPath, "mime", docMime, "media_id", mediaID) if isArchiveDocumentPath(docPath) { return NewResult(fmt.Sprintf( "Archive file available at %s. read_document does not analyze archive containers directly. Use exec to inspect or extract it, for example: unzip -l %q or unzip -q %q -d , then use list_files/read_file on extracted files.", - docPath, docPath, docPath, + displayPath, displayPath, displayPath, )) } // Read document file. data, err := os.ReadFile(docPath) if err != nil { - return ErrorResult(fmt.Sprintf("Failed to read document file: %v", err)) + slog.Warn("read_document: file read failed", + "path", displayPath, + "error", tracing.RedactText(ctx, err.Error()), + ) + return ErrorResult(fmt.Sprintf("Failed to read document file %s.", displayPath)) } slog.Info("read_document: file loaded", "size_bytes", len(data)) if len(data) > documentMaxBytes { @@ -169,7 +180,11 @@ func (t *ReadDocumentTool) Execute(ctx context.Context, args map[string]any) *Re // handing it to a subprocess. A rejection routes to vision rather than // erroring, preserving today's behavior for MediaRef-derived paths. if safePath, verr := t.validateExecPath(ctx, docPath); verr != nil { - slog.Warn("security.read_document_local_path_rejected", "path", docPath, "reason", verr.Error()) + slog.Warn( + "security.read_document_local_path_rejected", + "path", displayPath, + "reason", tracing.RedactText(ctx, verr.Error()), + ) } else if text, err := t.localParser.Extract(ctx, safePath, docMime); err == nil { slog.Info("read_document: local extraction hit", "mime", docMime, "bytes", len(text)) return NewResult(text) // no Provider/Model/Usage => no LLM spend @@ -203,6 +218,35 @@ func (t *ReadDocumentTool) Execute(ctx context.Context, args map[string]any) *Re return result } +// logicalDocumentDisplayPath returns only model-safe paths. Resolved host +// paths remain runtime-only even when tracing redaction is not configured. +func logicalDocumentDisplayPath(ctx context.Context, resolvedPath, requestedPath string) string { + requested := path.Clean(strings.ReplaceAll(strings.TrimSpace(requestedPath), "\\", "/")) + if requestedPath != "" && requested != "." && requested != ".." && + !strings.HasPrefix(requested, "../") && !path.IsAbs(requested) { + return requested + } + + if inputRoot := DelegationArtifactInputsFromCtx(ctx); inputRoot != "" { + if relative, ok := artifactRelativeToRoot(inputRoot, resolvedPath); ok && + relative != "." { + return path.Join("inputs", relative) + } + } + if workspace := ToolWorkspaceFromCtx(ctx); workspace != "" { + if relative, ok := artifactRelativeToRoot(workspace, resolvedPath); ok && + relative != "." { + return relative + } + } + + name := filepath.Base(resolvedPath) + if name == "." || name == string(filepath.Separator) || name == "" { + return "document" + } + return name +} + // validateExecPath confirms a resolved document path is workspace-confined // before it is passed to a local extractor subprocess. It reuses the same // allow/deny resolution as the explicit-path argument branch so MediaRef-derived @@ -217,5 +261,8 @@ func (t *ReadDocumentTool) validateExecPath(ctx context.Context, path string) (s if err := checkDeniedPath(resolved, workspace, nil); err != nil { return "", err } + if err := ValidateRegularFileForRead(resolved); err != nil { + return "", fmt.Errorf("document path is not a safe regular file: %w", err) + } return resolved, nil } diff --git a/internal/tools/read_document_resolve.go b/internal/tools/read_document_resolve.go index d32ceed1..1b54825e 100644 --- a/internal/tools/read_document_resolve.go +++ b/internal/tools/read_document_resolve.go @@ -5,7 +5,6 @@ import ( "encoding/base64" "fmt" "log/slog" - "os" "path/filepath" "strings" @@ -15,6 +14,9 @@ import ( // resolveDocumentFile finds the document file path from an explicit workspace // path or from context MediaRefs. func (t *ReadDocumentTool) resolveDocumentFile(ctx context.Context, mediaID, docPath string) (path, mime string, err error) { + if strings.TrimSpace(mediaID) != "" && strings.TrimSpace(docPath) != "" { + return "", "", fmt.Errorf("specify either media_id or path, not both") + } if docPath != "" { p, err := resolveDocumentPathArg(ctx, docPath) if err != nil { @@ -28,16 +30,11 @@ func (t *ReadDocumentTool) resolveDocumentFile(ctx context.Context, mediaID, doc return "", "", fmt.Errorf("no documents available in this conversation. The user may not have sent a document.") } - if strings.Contains(mediaID, "<") || strings.Contains(mediaID, "media:") { - slog.Debug("read_document: sanitizing tag-like media_id", "raw", mediaID) - mediaID = "" - } - // Find specific media_id or use most recent document. var ref *providers.MediaRef if mediaID != "" { for i := range refs { - if documentRefMatches(refs[i], mediaID) { + if refs[i].ID == mediaID { ref = &refs[i] break } @@ -52,6 +49,7 @@ func (t *ReadDocumentTool) resolveDocumentFile(ctx context.Context, mediaID, doc // Prefer persisted workspace path; fall back to legacy .media/ lookup. p := ref.Path + loadedLegacy := false if p == "" { var err error if t.mediaLoader == nil { @@ -61,6 +59,16 @@ func (t *ReadDocumentTool) resolveDocumentFile(ctx context.Context, mediaID, doc if err != nil { return "", "", fmt.Errorf("document file not found: %v", err) } + loadedLegacy = true + } + + if loadedLegacy { + p, err = resolveLoadedMediaRefPath(ctx, t.mediaLoader, p, "document") + } else { + p, err = resolveStructuredMediaRefPath(ctx, p, "document") + } + if err != nil { + return "", "", err } // Determine MIME type: prefer ref's stored MIME, fall back to extension. @@ -73,50 +81,7 @@ func (t *ReadDocumentTool) resolveDocumentFile(ctx context.Context, mediaID, doc } func resolveDocumentPathArg(ctx context.Context, path string) (string, error) { - workspace := ToolWorkspaceFromCtx(ctx) - resolved, err := resolvePathWithAllowed(path, workspace, effectiveRestrict(ctx, true), allowedWithTeamWorkspace(ctx, nil)) - if err != nil { - return "", fmt.Errorf("invalid document path: %w", err) - } - if err := checkDeniedPath(resolved, workspace, nil); err != nil { - return "", err - } - if info, err := os.Stat(resolved); err != nil { - return "", fmt.Errorf("failed to stat document file: %w", err) - } else if info.IsDir() { - return "", fmt.Errorf("document path is a directory: %s", path) - } - return resolved, nil -} - -func documentRefMatches(ref providers.MediaRef, mediaID string) bool { - if ref.ID == mediaID { - return true - } - if ref.Path == "" { - return false - } - want := strings.ToLower(filepath.Base(mediaID)) - got := strings.ToLower(filepath.Base(ref.Path)) - if want == got { - return true - } - return strings.EqualFold(stripUploadShortID(got), want) -} - -func stripUploadShortID(name string) string { - ext := filepath.Ext(name) - stem := strings.TrimSuffix(name, ext) - idx := strings.LastIndexByte(stem, '-') - if idx < 0 || len(stem)-idx-1 != 8 { - return name - } - for _, r := range stem[idx+1:] { - if !((r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F')) { - return name - } - } - return stem[:idx] + ext + return resolveStructuredMediaPath(ctx, path, "document") } func isArchiveDocumentPath(path string) bool { diff --git a/internal/tools/read_document_resolve_test.go b/internal/tools/read_document_resolve_test.go index 172ac87f..3f2aafe5 100644 --- a/internal/tools/read_document_resolve_test.go +++ b/internal/tools/read_document_resolve_test.go @@ -42,9 +42,15 @@ func TestResolveDocumentFileAcceptsWorkspacePath(t *testing.T) { } } -func TestResolveDocumentFileMatchesUploadedFilenameAlias(t *testing.T) { +func TestResolveDocumentFileRequiresExactMediaID(t *testing.T) { workspace := t.TempDir() docPath := filepath.Join(workspace, ".uploads", "codex-9c8914a5.zip") + if err := os.MkdirAll(filepath.Dir(docPath), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(docPath, []byte("zip bytes"), 0o644); err != nil { + t.Fatal(err) + } refs := []providers.MediaRef{{ ID: uuid.NewString(), Kind: "document", @@ -53,13 +59,22 @@ func TestResolveDocumentFileMatchesUploadedFilenameAlias(t *testing.T) { }} tool := NewReadDocumentTool(nil, nil) - ctx := WithMediaDocRefs(context.Background(), refs) - gotPath, gotMime, err := tool.resolveDocumentFile(ctx, "codex.zip", "") - if err != nil { - t.Fatalf("resolveDocumentFile returned error: %v", err) + ctx := WithToolWorkspace(context.Background(), workspace) + ctx = WithMediaDocRefs(ctx, refs) + if gotPath, _, err := tool.resolveDocumentFile(ctx, "codex.zip", ""); err == nil { + t.Fatalf("filename alias resolved to %q, want exact media_id error", gotPath) } - if gotPath != docPath { - t.Fatalf("path = %q, want %q", gotPath, docPath) + + gotPath, gotMime, err := tool.resolveDocumentFile(ctx, refs[0].ID, "") + if err != nil { + t.Fatalf("exact media_id returned error: %v", err) + } + wantPath, err := filepath.EvalSymlinks(docPath) + if err != nil { + t.Fatal(err) + } + if gotPath != wantPath { + t.Fatalf("path = %q, want %q", gotPath, wantPath) } if gotMime != "application/zip" { t.Fatalf("mime = %q, want application/zip", gotMime) @@ -84,19 +99,85 @@ func TestResolveDocumentFileInvalidMediaIDReturnsError(t *testing.T) { } func TestResolveDocumentFileOmittedMediaIDUsesLastRef(t *testing.T) { + workspace := t.TempDir() + oldPath := filepath.Join(workspace, ".uploads", "old.pdf") + latestPath := filepath.Join(workspace, ".uploads", "latest.pdf") + if err := os.MkdirAll(filepath.Dir(oldPath), 0o755); err != nil { + t.Fatal(err) + } + for _, path := range []string{oldPath, latestPath} { + if err := os.WriteFile(path, []byte("pdf"), 0o644); err != nil { + t.Fatal(err) + } + } refs := []providers.MediaRef{ - {ID: uuid.NewString(), Kind: "document", Path: "/workspace/.uploads/old.pdf", MimeType: "application/pdf"}, - {ID: uuid.NewString(), Kind: "document", Path: "/workspace/.uploads/latest.pdf", MimeType: "application/pdf"}, + {ID: uuid.NewString(), Kind: "document", Path: oldPath, MimeType: "application/pdf"}, + {ID: uuid.NewString(), Kind: "document", Path: latestPath, MimeType: "application/pdf"}, } tool := NewReadDocumentTool(nil, nil) - ctx := WithMediaDocRefs(context.Background(), refs) + ctx := WithToolWorkspace(context.Background(), workspace) + ctx = WithMediaDocRefs(ctx, refs) gotPath, _, err := tool.resolveDocumentFile(ctx, "", "") if err != nil { t.Fatalf("resolveDocumentFile returned error: %v", err) } - if gotPath != refs[1].Path { - t.Fatalf("path = %q, want most recent %q", gotPath, refs[1].Path) + wantPath, err := filepath.EvalSymlinks(refs[1].Path) + if err != nil { + t.Fatal(err) + } + if gotPath != wantPath { + t.Fatalf("path = %q, want most recent %q", gotPath, wantPath) + } +} + +func TestResolveDocumentFileRejectsRefPathOutsideWorkspace(t *testing.T) { + workspace := t.TempDir() + outsideDir := t.TempDir() + outsidePath := filepath.Join(outsideDir, "secret.pdf") + if err := os.WriteFile(outsidePath, []byte("secret"), 0o644); err != nil { + t.Fatal(err) + } + ref := providers.MediaRef{ + ID: uuid.NewString(), + Kind: "document", + Path: outsidePath, + MimeType: "application/pdf", + } + + ctx := WithToolWorkspace(context.Background(), workspace) + ctx = WithToolTeamWorkspace(ctx, outsideDir) + ctx = WithMediaDocRefs(ctx, []providers.MediaRef{ref}) + gotPath, _, err := NewReadDocumentTool(nil, nil).resolveDocumentFile(ctx, ref.ID, "") + if err == nil { + t.Fatalf("outside ref path resolved to %q, want containment error", gotPath) + } +} + +func TestResolveDocumentFileAcceptsDelegationInputRef(t *testing.T) { + ctx, inputs, _ := delegationArtifactToolContext(t) + docPath := filepath.Join(inputs, "brief.pdf") + if err := os.WriteFile(docPath, []byte("pdf"), 0o440); err != nil { + t.Fatal(err) + } + ref := providers.MediaRef{ + ID: uuid.NewString(), + Kind: "document", + Path: "inputs/brief.pdf", + MimeType: "application/pdf", + } + ctx = WithMediaDocRefs(ctx, []providers.MediaRef{ref}) + + gotPath, _, err := NewReadDocumentTool(nil, nil).resolveDocumentFile(ctx, ref.ID, "") + if err != nil { + t.Fatalf("delegation input ref returned error: %v", err) + } + wantPath, err := filepath.EvalSymlinks(docPath) + if err != nil { + t.Fatal(err) + } + if gotPath != wantPath { + t.Fatalf("path = %q, want staged input %q", gotPath, wantPath) } } @@ -121,7 +202,24 @@ func TestReadDocumentArchiveReturnsExecHint(t *testing.T) { if result.IsError { t.Fatalf("expected archive hint, got error: %s", result.ForLLM) } - if !strings.Contains(result.ForLLM, "unzip -l") || !strings.Contains(result.ForLLM, docPath) { - t.Fatalf("expected unzip hint with path, got: %s", result.ForLLM) + if !strings.Contains(result.ForLLM, "unzip -l") || + !strings.Contains(result.ForLLM, ".uploads/codex-9c8914a5.zip") { + t.Fatalf("expected unzip hint with logical path, got: %s", result.ForLLM) + } + if strings.Contains(result.ForLLM, workspace) || strings.Contains(result.ForLLM, docPath) { + t.Fatalf("archive hint leaked physical workspace path: %s", result.ForLLM) + } +} + +func TestReadDocumentRejectsMediaIDAndPathTogether(t *testing.T) { + tool := NewReadDocumentTool(nil, nil) + result := tool.Execute(context.Background(), map[string]any{ + "prompt": "Inspect", + "media_id": uuid.NewString(), + "path": ".uploads/document.pdf", + }) + + if !result.IsError || !strings.Contains(result.ForLLM, "either media_id or path") { + t.Fatalf("result = %#v, want mutually exclusive argument error", result) } } diff --git a/internal/tools/read_image.go b/internal/tools/read_image.go index 8664184d..cc891056 100644 --- a/internal/tools/read_image.go +++ b/internal/tools/read_image.go @@ -17,6 +17,7 @@ import ( // --- Context helpers for media images --- const ctxMediaImages toolContextKey = "tool_media_images" +const ctxMediaImageRefs toolContextKey = "tool_media_image_refs" // WithMediaImages stores base64-encoded images in context for read_image tool access. func WithMediaImages(ctx context.Context, images []providers.ImageContent) context.Context { @@ -29,6 +30,18 @@ func MediaImagesFromCtx(ctx context.Context) []providers.ImageContent { return v } +// WithMediaImageRefs stores image MediaRefs in chronological order for exact +// media_id resolution by image tools. +func WithMediaImageRefs(ctx context.Context, refs []providers.MediaRef) context.Context { + return context.WithValue(ctx, ctxMediaImageRefs, refs) +} + +// MediaImageRefsFromCtx retrieves image MediaRefs available to the current run. +func MediaImageRefsFromCtx(ctx context.Context) []providers.MediaRef { + v, _ := ctx.Value(ctxMediaImageRefs).([]providers.MediaRef) + return v +} + // --- ReadImageTool --- // visionProviderPriority is the order in which providers are tried for vision. @@ -76,7 +89,11 @@ func (t *ReadImageTool) Parameters() map[string]any { }, "path": map[string]any{ "type": "string", - "description": "Optional file path to an image in the workspace. Use this for generated images or attachments. If omitted, analyzes images from the conversation.", + "description": "Optional logical file path to an image in the workspace (for example .uploads/photo.jpg or inputs/photo.jpg). Never reconstruct an absolute workspace path.", + }, + "media_id": map[string]any{ + "type": "string", + "description": "Optional exact media ID from a tag for in-process tool execution. Claude CLI/MCP callers should use the tag's logical path.", }, "url": map[string]any{ "type": "string", @@ -98,10 +115,14 @@ func (t *ReadImageTool) Execute(ctx context.Context, args map[string]any) *Resul imgPath, _ := args["path"].(string) imgURL, _ := args["url"].(string) + mediaID, _ := args["media_id"].(string) - if imgPath != "" && imgURL != "" { + if imgPath != "" && imgURL != "" && mediaID == "" { return ErrorResult("Both 'path' and 'url' parameters cannot be specified. Choose only one.") } + if mediaID != "" && (imgPath != "" || imgURL != "") { + return ErrorResult("Only one of 'path', 'url', or 'media_id' may be specified.") + } // If path is provided, load image from workspace file images := MediaImagesFromCtx(ctx) @@ -111,6 +132,19 @@ func (t *ReadImageTool) Execute(ctx context.Context, args map[string]any) *Resul return ErrorResult(err.Error()) } images = fileImages + } else if mediaID != "" { + ref, err := resolveImageMediaRef(ctx, mediaID) + if err != nil { + return ErrorResult(err.Error()) + } + if ref.Path == "" { + return ErrorResult(fmt.Sprintf("image media_id %q has no accessible workspace path", mediaID)) + } + fileImages, err := t.loadImageFromPath(ctx, ref.Path) + if err != nil { + return ErrorResult(err.Error()) + } + images = fileImages } else if imgURL != "" { if _, _, err := security.Validate(imgURL); err != nil { return ErrorResult(fmt.Sprintf("Invalid image URL: %v", err)) @@ -152,6 +186,22 @@ func (t *ReadImageTool) Execute(ctx context.Context, args map[string]any) *Resul return result } +func resolveImageMediaRef(ctx context.Context, mediaID string) (providers.MediaRef, error) { + refs := MediaImageRefsFromCtx(ctx) + if len(refs) == 0 { + return providers.MediaRef{}, fmt.Errorf("no image media references available in this conversation") + } + if mediaID == "latest" { + return refs[len(refs)-1], nil + } + for _, ref := range refs { + if ref.ID == mediaID { + return ref, nil + } + } + return providers.MediaRef{}, fmt.Errorf("image media_id %q not found in this conversation", mediaID) +} + // callProvider dispatches the vision call using provider.Chat(). func (t *ReadImageTool) callProvider(ctx context.Context, cp credentialProvider, providerName, model string, params map[string]any) ([]byte, *providers.Usage, error) { prompt := GetParamString(params, "prompt", "Describe this image in detail.") @@ -235,13 +285,8 @@ func (t *ReadImageTool) loadImageFromPath(ctx context.Context, path string) ([]p return nil, fmt.Errorf("unsupported image format: %s (supported: jpg, png, gif, webp, bmp)", ext) } - // Resolve path within workspace (respect workspace restriction). - workspace := ToolWorkspaceFromCtx(ctx) - resolved, err := resolvePathWithAllowed(path, workspace, effectiveRestrict(ctx, true), allowedWithTeamWorkspace(ctx, nil)) + resolved, err := resolveStructuredMediaPath(ctx, path, "image") if err != nil { - return nil, fmt.Errorf("invalid image path: %w", err) - } - if err := checkDeniedPath(resolved, workspace, nil); err != nil { return nil, err } diff --git a/internal/tools/read_image_test.go b/internal/tools/read_image_test.go index 87ee177a..91ae5d87 100644 --- a/internal/tools/read_image_test.go +++ b/internal/tools/read_image_test.go @@ -42,6 +42,25 @@ func TestReadImage_PrivateURL_Error(t *testing.T) { } } +func TestResolveImageMediaRefRequiresExactID(t *testing.T) { + ctx := WithMediaImageRefs(context.Background(), []providers.MediaRef{ + {ID: "first-id", Kind: "image", Path: ".uploads/first.png"}, + {ID: "second-id", Kind: "image", Path: ".uploads/second.png"}, + }) + + got, err := resolveImageMediaRef(ctx, "first-id") + if err != nil || got.ID != "first-id" { + t.Fatalf("exact media ID = %#v, %v", got, err) + } + latest, err := resolveImageMediaRef(ctx, "latest") + if err != nil || latest.ID != "second-id" { + t.Fatalf("latest media ID = %#v, %v", latest, err) + } + if _, err := resolveImageMediaRef(ctx, "missing-id"); err == nil { + t.Fatal("unknown media ID unexpectedly resolved") + } +} + func TestReadImage_AnthropicURL_Error(t *testing.T) { tool := NewReadImageTool(nil) diff --git a/internal/tools/read_video.go b/internal/tools/read_video.go index 48b1aaba..5f660d90 100644 --- a/internal/tools/read_video.go +++ b/internal/tools/read_video.go @@ -67,7 +67,7 @@ func (t *ReadVideoTool) Name() string { return "read_video" } func (t *ReadVideoTool) Description() string { return "Analyze video files attached to the conversation. " + "Use when you see tags and need to describe, summarize, or analyze video content. " + - "Specify what you want to extract or analyze." + "A workspace-relative path such as inputs/clip.mp4 may also be provided. Specify what you want to extract or analyze." } func (t *ReadVideoTool) Parameters() map[string]any { @@ -86,6 +86,10 @@ func (t *ReadVideoTool) Parameters() map[string]any { "type": "string", "description": "Optional URL to a video file. Use this to analyze videos hosted online.", }, + "path": map[string]any{ + "type": "string", + "description": "Optional workspace-relative video path. Delegated inputs use inputs/.", + }, }, "required": []string{"prompt"}, } @@ -98,9 +102,19 @@ func (t *ReadVideoTool) Execute(ctx context.Context, args map[string]any) *Resul } mediaID, _ := args["media_id"].(string) videoURL, _ := args["url"].(string) + videoArg, _ := args["path"].(string) - if mediaID != "" && videoURL != "" { - return ErrorResult("Both 'media_id' and 'url' parameters cannot be specified. Choose only one.") + sourceCount := 0 + for _, source := range []string{mediaID, videoURL, videoArg} { + if source != "" { + sourceCount++ + } + } + if sourceCount > 1 { + if videoArg == "" && mediaID != "" && videoURL != "" { + return ErrorResult("Both 'media_id' and 'url' parameters cannot be specified. Choose only one.") + } + return ErrorResult("Only one of 'media_id', 'url', or 'path' may be specified.") } var data []byte @@ -118,15 +132,25 @@ func (t *ReadVideoTool) Execute(ctx context.Context, args map[string]any) *Resul ext := filepath.Ext(validatedURL.Path) videoMime = mimeFromVideoExt(ext) } else { - videoPath, mime, err := t.resolveVideoFile(ctx, mediaID) + var videoPath, mime string + var err error + if videoArg != "" { + videoPath, err = resolveStructuredMediaPath(ctx, videoArg, "video") + mime = mimeFromVideoExt(filepath.Ext(videoPath)) + } else { + videoPath, mime, err = t.resolveVideoFile(ctx, mediaID) + } if err != nil { return ErrorResult(err.Error()) } videoMime = mime - slog.Info("read_video: resolved file", "path", videoPath, "mime", videoMime, "media_id", mediaID) + slog.Info("read_video: resolved file", "mime", videoMime, "media_id", mediaID, "logical_path", videoArg) fileData, err := os.ReadFile(videoPath) if err != nil { + if videoArg != "" && IsDelegationArtifactRun(ctx) { + return ErrorResult("Failed to read delegation video input") + } return ErrorResult(fmt.Sprintf("Failed to read video file: %v", err)) } slog.Info("read_video: file loaded", "size_bytes", len(fileData)) diff --git a/internal/tools/read_video_resolve.go b/internal/tools/read_video_resolve.go index 0a6d0a0a..548fd799 100644 --- a/internal/tools/read_video_resolve.go +++ b/internal/tools/read_video_resolve.go @@ -44,6 +44,7 @@ func (t *ReadVideoTool) resolveVideoFile(ctx context.Context, mediaID string) (p // Prefer persisted workspace path; fall back to legacy .media/ lookup. p := ref.Path + loadedLegacy := false if p == "" { var err error if t.mediaLoader == nil { @@ -53,6 +54,16 @@ func (t *ReadVideoTool) resolveVideoFile(ctx context.Context, mediaID string) (p if err != nil { return "", "", fmt.Errorf("video file not found: %v", err) } + loadedLegacy = true + } + + if loadedLegacy { + p, err = resolveLoadedMediaRefPath(ctx, t.mediaLoader, p, "video") + } else { + p, err = resolveStructuredMediaRefPath(ctx, p, "video") + } + if err != nil { + return "", "", err } mime = ref.MimeType diff --git a/internal/tools/sandbox_mount_test.go b/internal/tools/sandbox_mount_test.go index 1232c358..dcdcc4f2 100644 --- a/internal/tools/sandbox_mount_test.go +++ b/internal/tools/sandbox_mount_test.go @@ -3,7 +3,9 @@ package tools import ( "context" "os" + "path" "path/filepath" + "strings" "testing" "time" @@ -18,12 +20,14 @@ type recordingSandboxManager struct { key string workspace string cfg *sandbox.Config + getOpts sandbox.GetOpts } -func (m *recordingSandboxManager) Get(ctx context.Context, key string, workspace string, cfg *sandbox.Config) (sandbox.Sandbox, error) { +func (m *recordingSandboxManager) Get(ctx context.Context, key string, workspace string, cfg *sandbox.Config, opts ...sandbox.GetOption) (sandbox.Sandbox, error) { m.key = key m.workspace = workspace m.cfg = cfg + m.getOpts = sandbox.ApplyGetOpts(opts) if m.sandbox == nil { m.sandbox = &recordingSandbox{} } @@ -153,6 +157,40 @@ func TestCredentialedExecSandboxUsesEffectiveWorkspaceMountAndContainerCwd(t *te } } +func TestCredentialedExecSandboxMountsCurrentDelegationExchange(t *testing.T) { + ctx, outputs := delegatedExecSandboxContext(t) + ctx = WithToolSandboxKey(ctx, "delegated-credentialed-session") + manager := &recordingSandboxManager{} + tool := NewSandboxedExecTool(outputs, true, manager) + + result := tool.executeCredentialedSandbox( + ctx, + "/usr/bin/gh", + []string{"api", "user"}, + outputs, + "delegated-credentialed-session", + map[string]string{"GH_TOKEN": "secret"}, + 30*time.Second, + ) + + if result.IsError { + t.Fatalf("credentialed delegated sandbox exec failed: %s", result.ForLLM) + } + if manager.getOpts.WorkspaceAccessOverride == nil || + *manager.getOpts.WorkspaceAccessOverride != sandbox.AccessRW { + t.Fatalf("delegation output access override = %#v, want rw", manager.getOpts.WorkspaceAccessOverride) + } + if len(manager.getOpts.ReadOnlyMounts) != 1 { + t.Fatalf("read-only mounts = %#v, want one inputs mount", manager.getOpts.ReadOnlyMounts) + } + mount := manager.getOpts.ReadOnlyMounts[0] + if mount.Name != "inputs" || + mount.Destination != path.Join(sandbox.DefaultContainerWorkdir, "inputs") || + filepath.Base(mount.HostPath) != "inputs" { + t.Fatalf("delegation input mount = %#v", mount) + } +} + func TestCredentialedExecSandboxWorkingDirResolvesInsideTenantWorkspace(t *testing.T) { globalWorkspace := t.TempDir() tenantWorkspace := filepath.Join(globalWorkspace, "tenants", "acme") @@ -244,6 +282,97 @@ func TestSandboxFileToolsUseEffectiveWorkspaceMount(t *testing.T) { } } +func TestAcquireToolSandboxMountsOnlyCurrentDelegationInputs(t *testing.T) { + tenantWorkspace := t.TempDir() + delegationID := uuid.New() + root := filepath.Join(tenantWorkspace, "collaboration", "delegations", delegationID.String()) + inputs := filepath.Join(root, "inputs") + outputs := filepath.Join(root, "outputs") + if err := os.MkdirAll(inputs, 0o750); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(outputs, 0o750); err != nil { + t.Fatal(err) + } + cfg := sandbox.DefaultConfig() + cfg.Mode = sandbox.ModeAll + cfg.WorkspaceAccess = sandbox.AccessNone + ctx := WithDelegationID(context.Background(), delegationID.String()) + ctx = WithDelegationArtifactInputs(ctx, inputs) + ctx = WithToolWorkspace(ctx, outputs) + ctx = WithSandboxConfig(ctx, &cfg) + + mgr := &recordingSandboxManager{} + if _, err := acquireToolSandbox(ctx, mgr, "delegate-session", outputs); err != nil { + t.Fatalf("acquireToolSandbox: %v", err) + } + if len(mgr.getOpts.ReadOnlyMounts) != 1 { + t.Fatalf("read-only mounts = %#v, want one input mount", mgr.getOpts.ReadOnlyMounts) + } + if mgr.getOpts.WorkspaceAccessOverride == nil || + *mgr.getOpts.WorkspaceAccessOverride != sandbox.AccessRW { + t.Fatalf("delegation output access override = %#v, want rw", mgr.getOpts.WorkspaceAccessOverride) + } + mount := mgr.getOpts.ReadOnlyMounts[0] + wantInputs, err := filepath.EvalSymlinks(inputs) + if err != nil { + t.Fatal(err) + } + if mount.Name != "inputs" || mount.HostPath != wantInputs || mount.Destination != "/workspace/inputs" { + t.Fatalf("input mount = %#v", mount) + } +} + +func TestAcquireToolSandboxRejectsExchangePathNotBoundToDelegationID(t *testing.T) { + root := filepath.Join(t.TempDir(), "collaboration", "delegations", uuid.NewString()) + inputs := filepath.Join(root, "inputs") + outputs := filepath.Join(root, "outputs") + if err := os.MkdirAll(inputs, 0o750); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(outputs, 0o750); err != nil { + t.Fatal(err) + } + + ctx := WithDelegationID(context.Background(), uuid.NewString()) + ctx = WithDelegationArtifactInputs(ctx, inputs) + ctx = WithToolWorkspace(ctx, outputs) + mgr := &recordingSandboxManager{} + if _, err := acquireToolSandbox(ctx, mgr, "delegate-session", outputs); err == nil { + t.Fatal("forged exchange path unexpectedly mounted") + } + if mgr.key != "" { + t.Fatal("sandbox manager called for forged exchange path") + } +} + +func TestDelegationInputMutationRejectedBeforeSandboxAcquisition(t *testing.T) { + ctx, _, outputs := delegationArtifactToolContext(t) + ctx = WithToolSandboxKey(ctx, "delegate-session") + + writeManager := &recordingSandboxManager{} + writeResult := NewSandboxedWriteFileTool(outputs, true, writeManager).Execute(ctx, map[string]any{ + "path": "inputs/source.txt", "content": "mutated", + }) + if !writeResult.IsError || !strings.Contains(writeResult.ForLLM, "read-only") { + t.Fatalf("write input result = %#v", writeResult) + } + if writeManager.key != "" { + t.Fatalf("write acquired sandbox before rejecting input mutation") + } + + editManager := &recordingSandboxManager{} + editResult := NewSandboxedEditTool(outputs, true, editManager).Execute(ctx, map[string]any{ + "path": "inputs/source.txt", "old_string": "a", "new_string": "b", + }) + if !editResult.IsError || !strings.Contains(editResult.ForLLM, "read-only") { + t.Fatalf("edit input result = %#v", editResult) + } + if editManager.key != "" { + t.Fatalf("edit acquired sandbox before rejecting input mutation") + } +} + type sandboxMountSecureCLIStore struct { binary *store.SecureCLIBinary } diff --git a/internal/tools/sandbox_utils.go b/internal/tools/sandbox_utils.go index 068e442a..3775bc20 100644 --- a/internal/tools/sandbox_utils.go +++ b/internal/tools/sandbox_utils.go @@ -4,13 +4,22 @@ import ( "context" "fmt" "log/slog" + "os" "path" "path/filepath" "strings" + "github.com/google/uuid" + + "github.com/nextlevelbuilder/goclaw/internal/sandbox" "github.com/nextlevelbuilder/goclaw/internal/store" ) +const ( + delegationArtifactInputsDirName = "inputs" + delegationArtifactOutputsDirName = "outputs" +) + // SandboxCwd maps the current effective workspace (from context) to its // corresponding path inside the sandbox container. The sandbox mounts the // global workspace root at containerBase (usually "/workspace"). This function @@ -59,6 +68,83 @@ func canonicalSandboxWorkspace(workspace string) string { return clean } +func sandboxContainerWorkdir(ctx context.Context) string { + if cfg := SandboxConfigFromCtx(ctx); cfg != nil { + return cfg.ContainerWorkdir() + } + return sandbox.DefaultContainerWorkdir +} + +// acquireToolSandbox attaches the delegation input exchange only to the exact +// delegated run that owns it. The input/output sibling check prevents a forged +// context from turning this runtime-only option into general host mounting. +func acquireToolSandbox( + ctx context.Context, + manager sandbox.Manager, + key, workspace string, +) (sandbox.Sandbox, error) { + cfg := SandboxConfigFromCtx(ctx) + inputRoot := DelegationArtifactInputsFromCtx(ctx) + if inputRoot == "" { + return manager.Get(ctx, key, workspace, cfg) + } + delegationID, err := uuid.Parse(DelegationIDFromCtx(ctx)) + if err != nil { + return nil, fmt.Errorf("delegation artifact sandbox context is invalid") + } + + outputRoot := ToolWorkspaceFromCtx(ctx) + inputCanonical, err := canonicalRealDirectory(inputRoot) + if err != nil { + return nil, fmt.Errorf("delegation artifact input mount is unavailable") + } + outputCanonical, err := canonicalRealDirectory(outputRoot) + if err != nil { + return nil, fmt.Errorf("delegation artifact output mount is unavailable") + } + if filepath.Base(inputCanonical) != delegationArtifactInputsDirName || + filepath.Base(outputCanonical) != delegationArtifactOutputsDirName || + filepath.Dir(inputCanonical) != filepath.Dir(outputCanonical) { + return nil, fmt.Errorf("delegation artifact sandbox context is invalid") + } + exchangeRoot := filepath.Dir(inputCanonical) + if filepath.Base(exchangeRoot) != delegationID.String() || + filepath.Base(filepath.Dir(exchangeRoot)) != "delegations" || + filepath.Base(filepath.Dir(filepath.Dir(exchangeRoot))) != "collaboration" { + return nil, fmt.Errorf("delegation artifact sandbox context is invalid") + } + + mount := sandbox.ReadOnlyMount{ + Name: "inputs", + HostPath: inputCanonical, + Destination: path.Join(sandboxContainerWorkdir(ctx), "inputs"), + } + return manager.Get( + ctx, + key, + workspace, + cfg, + sandbox.WithWorkspaceAccessOverride(sandbox.AccessRW), + sandbox.WithReadOnlyMounts(mount), + ) +} + +func canonicalRealDirectory(raw string) (string, error) { + if raw == "" || !filepath.IsAbs(raw) { + return "", fmt.Errorf("directory path must be absolute") + } + clean := filepath.Clean(raw) + info, err := os.Lstat(clean) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return "", fmt.Errorf("directory path is not a real directory") + } + real, err := filepath.EvalSymlinks(clean) + if err != nil { + return "", fmt.Errorf("directory path is not canonical") + } + return filepath.Clean(real), nil +} + func sandboxCwdForHostPath(hostCwd, mountWorkspace, containerBase string) (string, error) { if hostCwd == "" { hostCwd = mountWorkspace diff --git a/internal/tools/send_file.go b/internal/tools/send_file.go index caff4ad4..258f9835 100644 --- a/internal/tools/send_file.go +++ b/internal/tools/send_file.go @@ -80,6 +80,9 @@ func (t *SendFileTool) Parameters() map[string]any { // Execute resolves and validates the path, checks for duplicate delivery, then // returns a Result with Media populated for downstream pipeline delivery. func (t *SendFileTool) Execute(ctx context.Context, args map[string]any) *Result { + if IsDelegationArtifactRun(ctx) { + return ErrorResult("delegation files are published only after the delegated run completes") + } requests, err := parseSendFileRequests(args) if err != nil { return ErrorResult(err.Error()) diff --git a/internal/tools/shell.go b/internal/tools/shell.go index f0885cd0..f11914cd 100644 --- a/internal/tools/shell.go +++ b/internal/tools/shell.go @@ -27,6 +27,8 @@ const ( ExecDefaultTimeoutSeconds = 60 ExecMinTimeoutSeconds = 1 ExecMaxTimeoutSeconds = 3600 + + delegatedExecSandboxRequiredError = "delegated artifact exec requires an active sandbox manager and sandbox key; host execution is not allowed" ) // Dangerous command patterns organized into configurable deny groups. @@ -246,6 +248,14 @@ func (t *ExecTool) Execute(ctx context.Context, args map[string]any) *Result { return ErrorResult("command contains invalid NUL byte") } + // Delegation artifact runs are a physical isolation boundary. Reject before + // credential lookup, adapter preparation, approval prompts, or ordinary + // command handling unless execution can be routed into an identified sandbox. + if IsDelegationArtifactRun(ctx) && + (t.sandboxMgr == nil || ToolSandboxKeyFromCtx(ctx) == "") { + return ErrorResult(delegatedExecSandboxRequiredError) + } + // Normalize command before all deny checks: NFKC + zero-width strip prevents // Unicode-based pattern bypass while preserving functional command content. normalizedCommand := normalizeCommand(command) @@ -655,14 +665,17 @@ func (t *ExecTool) executeInSandbox(ctx context.Context, command, cwd, sandboxKe if err != nil { return ErrorResult(err.Error()) } - containerCwd, cwdErr := sandboxCwdForHostPath(cwd, mountWorkspace, sandbox.DefaultContainerWorkdir) + containerCwd, cwdErr := sandboxCwdForHostPath(cwd, mountWorkspace, sandboxContainerWorkdir(ctx)) if cwdErr != nil { return ErrorResult(fmt.Sprintf("sandbox path mapping: %v", cwdErr)) } - sb, err := t.sandboxMgr.Get(ctx, sandboxKey, mountWorkspace, SandboxConfigFromCtx(ctx)) + sb, err := acquireToolSandbox(ctx, t.sandboxMgr, sandboxKey, mountWorkspace) if err != nil { if errors.Is(err, sandbox.ErrSandboxDisabled) { + if IsDelegationArtifactRun(ctx) { + return ErrorResult(delegatedExecSandboxRequiredError) + } return t.executeOnHost(ctx, command, cwd) } // Docker unavailable (binary missing, daemon down) → fail closed. diff --git a/internal/tools/shell_delegation_boundary_test.go b/internal/tools/shell_delegation_boundary_test.go new file mode 100644 index 00000000..24e83345 --- /dev/null +++ b/internal/tools/shell_delegation_boundary_test.go @@ -0,0 +1,189 @@ +package tools + +import ( + "context" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" + + "github.com/google/uuid" + + "github.com/nextlevelbuilder/goclaw/internal/sandbox" + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +func TestExecDelegationArtifactRejectsNativeHostRead(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX shell command fixture") + } + + ctx, _, outputs := delegationArtifactToolContext(t) + outsideDir := t.TempDir() + outsideFile := filepath.Join(outsideDir, "outside.txt") + const outsideContents = "delegated-native-host-read" + if err := os.WriteFile(outsideFile, []byte(outsideContents), 0o600); err != nil { + t.Fatal(err) + } + + result := NewExecTool(outputs, true).Execute(ctx, map[string]any{ + "command": "cat " + strconv.Quote(outsideFile), + }) + + if !result.IsError { + t.Fatalf("delegated native exec read an outside host file: %#v", result) + } + if result.ForLLM != delegatedExecSandboxRequiredError { + t.Fatalf("error = %q, want %q", result.ForLLM, delegatedExecSandboxRequiredError) + } + if strings.Contains(result.ForLLM, outsideContents) { + t.Fatal("delegated native exec exposed outside file contents") + } +} + +func TestExecDelegationArtifactRejectsCredentialedHostPathBeforeLookup(t *testing.T) { + ctx, _, outputs := delegationArtifactToolContext(t) + secureCLIStore := newStubSecureCLIStore() + secureCLIStore.byName["delegated-cli"] = &store.SecureCLIBinary{ + BinaryName: "delegated-cli", + TimeoutSeconds: 30, + Enabled: true, + IsGlobal: true, + } + tool := NewExecTool(outputs, true) + tool.SetSecureCLIStore(secureCLIStore) + + result := tool.Execute(ctx, map[string]any{"command": "delegated-cli status"}) + + if !result.IsError || result.ForLLM != delegatedExecSandboxRequiredError { + t.Fatalf("delegated credentialed exec result = %#v, want stable sandbox-required error", result) + } + secureCLIStore.mu.Lock() + lookupCalls := secureCLIStore.lookupCalls + secureCLIStore.mu.Unlock() + if lookupCalls != 0 { + t.Fatalf("credential lookup calls = %d, want guard before credentialed routing", lookupCalls) + } +} + +func TestExecDelegationArtifactRejectsMissingSandboxKey(t *testing.T) { + ctx, _, outputs := delegationArtifactToolContext(t) + manager := &recordingSandboxManager{} + tool := NewSandboxedExecTool(outputs, true, manager) + + result := tool.Execute(ctx, map[string]any{"command": "echo should-not-run"}) + + if !result.IsError || result.ForLLM != delegatedExecSandboxRequiredError { + t.Fatalf("delegated exec without sandbox key = %#v, want stable sandbox-required error", result) + } + if manager.key != "" { + t.Fatalf("sandbox manager called with key %q despite missing context key", manager.key) + } +} + +func TestExecDelegationArtifactUsesSandboxWithManagerAndKey(t *testing.T) { + ctx, outputs := delegatedExecSandboxContext(t) + ctx = WithToolSandboxKey(ctx, "delegated-session") + manager := &recordingSandboxManager{} + tool := NewSandboxedExecTool(outputs, true, manager) + + result := tool.Execute(ctx, map[string]any{"command": "echo sandboxed"}) + + if result.IsError { + t.Fatalf("delegated sandbox exec failed: %s", result.ForLLM) + } + if manager.key != "delegated-session" { + t.Fatalf("sandbox manager key = %q, want delegated-session", manager.key) + } + if got := strings.Join(manager.sandbox.command, " "); got != "sh -c echo sandboxed" { + t.Fatalf("sandbox command = %q, want sandbox shell path", got) + } +} + +type disabledDelegationSandboxManager struct { + getCalls int +} + +func (m *disabledDelegationSandboxManager) Get(context.Context, string, string, *sandbox.Config, ...sandbox.GetOption) (sandbox.Sandbox, error) { + m.getCalls++ + return nil, sandbox.ErrSandboxDisabled +} + +func (*disabledDelegationSandboxManager) Release(context.Context, string) error { return nil } +func (*disabledDelegationSandboxManager) ReleaseAll(context.Context) error { return nil } +func (*disabledDelegationSandboxManager) Stop() {} +func (*disabledDelegationSandboxManager) Stats() map[string]any { return nil } + +func TestExecDelegationArtifactDoesNotFallbackWhenSandboxDisabled(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX touch command fixture") + } + + ctx, outputs := delegatedExecSandboxContext(t) + ctx = WithToolSandboxKey(ctx, "delegated-session") + marker := filepath.Join(t.TempDir(), "process-ran") + manager := &disabledDelegationSandboxManager{} + tool := NewSandboxedExecTool(outputs, true, manager) + + result := tool.Execute(ctx, map[string]any{ + "command": "touch " + strconv.Quote(marker), + }) + + if !result.IsError || result.ForLLM != delegatedExecSandboxRequiredError { + t.Fatalf("disabled delegated sandbox result = %#v, want stable sandbox-required error", result) + } + if manager.getCalls != 1 { + t.Fatalf("sandbox manager Get calls = %d, want 1", manager.getCalls) + } + if _, err := os.Stat(marker); !os.IsNotExist(err) { + t.Fatalf("delegated exec fell back to host; marker stat error = %v", err) + } +} + +func TestExecNonDelegatedSandboxDisabledStillFallsBackToHost(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX echo command fixture") + } + + workspace := t.TempDir() + canonicalWorkspace, err := filepath.EvalSymlinks(workspace) + if err != nil { + t.Fatal(err) + } + manager := &disabledDelegationSandboxManager{} + tool := NewSandboxedExecTool(canonicalWorkspace, true, manager) + ctx := WithToolSandboxKey(context.Background(), "ordinary-session") + + result := tool.Execute(ctx, map[string]any{"command": "echo ordinary-fallback"}) + + if result.IsError { + t.Fatalf("non-delegated sandbox fallback changed: %s", result.ForLLM) + } + if !strings.Contains(result.ForLLM, "ordinary-fallback") { + t.Fatalf("non-delegated host fallback output = %q", result.ForLLM) + } +} + +func delegatedExecSandboxContext(t *testing.T) (context.Context, string) { + t.Helper() + delegationID := uuid.New() + root := filepath.Join(t.TempDir(), "collaboration", "delegations", delegationID.String()) + inputs := filepath.Join(root, "inputs") + outputs := filepath.Join(root, "outputs") + if err := os.MkdirAll(inputs, 0o750); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(outputs, 0o750); err != nil { + t.Fatal(err) + } + canonicalOutputs, err := filepath.EvalSymlinks(outputs) + if err != nil { + t.Fatal(err) + } + ctx := WithDelegationID(context.Background(), delegationID.String()) + ctx = WithDelegationArtifactInputs(ctx, inputs) + ctx = WithToolWorkspace(ctx, canonicalOutputs) + return ctx, canonicalOutputs +} diff --git a/internal/tools/subagent.go b/internal/tools/subagent.go index 178eca1a..bab1bb70 100644 --- a/internal/tools/subagent.go +++ b/internal/tools/subagent.go @@ -1,39 +1,47 @@ // Package tools provides the subagent system for spawning child agent instances. // // Subagents run in background goroutines with restricted tool access. -// Key constraints from OpenClaw spec: -// - Depth limit: configurable maxSpawnDepth (default 3) -// - Max children per parent: configurable (default 8) -// - Auto-archive after configurable TTL (default 30 min) +// Key GoClaw constraints: +// - Depth limit: configurable maxSpawnDepth (default 1) +// - Max children per parent: configurable (default 5) +// - Max executing descendants per root agent: configurable (default 20) +// - Auto-archive after configurable TTL (default 60 min) // - Tool deny lists: ALWAYS_DENY + LEAF_DENY at max depth // - Results announced back to parent via message bus package tools import ( "context" + "errors" "sync" + "time" "github.com/google/uuid" "github.com/nextlevelbuilder/goclaw/internal/bus" + orchestration "github.com/nextlevelbuilder/goclaw/internal/childrun" "github.com/nextlevelbuilder/goclaw/internal/providers" "github.com/nextlevelbuilder/goclaw/internal/store" usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps" ) +var ErrSubagentLifecycleDrainTimeout = errors.New("subagent_lifecycle_drain_timeout") + // SubagentConfig configures the subagent system. type SubagentConfig struct { - MaxConcurrent int // max concurrent subagents (default 4) - MaxSpawnDepth int // max nesting depth (default 3) - MaxChildrenPerAgent int // max children per parent (default 8) - ArchiveAfterMinutes int // auto-archive completed tasks (default 30) + MaxConcurrent int // executing descendants per root agent (default 20) + MaxSpawnDepth int // max nesting depth (default 1) + MaxChildrenPerAgent int // max children per parent (default 5) + ArchiveAfterMinutes int // auto-archive completed tasks (default 60) MaxRetries int // max LLM call retries on error (default 2) Model string // model override for subagents (empty = inherit) } // Subagent task status constants. const ( + TaskStatusQueued = "queued" TaskStatusRunning = "running" + TaskStatusWaiting = string(orchestration.ChildRunWaitingChild) TaskStatusCompleted = "completed" TaskStatusFailed = "failed" TaskStatusCancelled = "cancelled" @@ -62,17 +70,30 @@ type SubagentTask struct { CreatedAt int64 `json:"createdAt"` CompletedAt int64 `json:"completedAt,omitempty"` Media []bus.MediaFile `json:"-"` // media files from tool results + Workspace string `json:"-"` // physical root used only to derive safe completion media paths + MediaPathPrefix string `json:"-"` // logical prefix, e.g. outputs/ in a delegation exchange OriginAgentID uuid.UUID `json:"-"` // parent agent UUID for usage caps and scoped tools OriginTenantID uuid.UUID `json:"-"` // parent's tenant for announce routing + RootAgentID uuid.UUID `json:"-"` + RootAgentKey string `json:"-"` + ParentTaskID string `json:"parentTaskId,omitempty"` OriginTraceID uuid.UUID `json:"-"` // parent trace for announce linking OriginRootSpanID uuid.UUID `json:"-"` // parent agent's root span ID - // OriginContextWindow and OriginMaxTokens are captured from the caller at - // spawn so a shared manager cannot mix budgets between agents. + // Capture caller-specific budgets because one process manager serves many agents. OriginContextWindow int `json:"-"` OriginMaxTokens int `json:"-"` cancelFunc context.CancelFunc `json:"-"` // per-task context cancel spawnConfig SubagentConfig `json:"-"` // resolved config at spawn time (per-agent override merged) dbID uuid.UUID `json:"-"` // persistent DB UUID (zero if not persisted) + admissionTicket *orchestration.ChildRunTicket +} + +// TaskScope is the authorization boundary for every in-memory task action. +// A task ID alone never authorizes lookup, cancellation, steering, or cleanup. +type TaskScope struct { + TenantID uuid.UUID + RootAgentID uuid.UUID + RootAgentKey string } // SubagentManager manages the lifecycle of spawned subagents. @@ -86,11 +107,22 @@ type SubagentManager struct { msgBus *bus.MessageBus // createTools builds a tool registry for subagents (without spawn/subagent tools). - createTools func() *Registry - announceQueue *AnnounceQueue // optional: batches announces with debounce - taskStore store.SubagentTaskStore // optional: persists tasks to DB (fire-and-forget) - usageCaps *usagecaps.Service - // Default agent budget used only when a task was created without caller context. + createTools func() *Registry + announceQueue *AnnounceQueue // optional: batches announces with debounce + taskStore store.SubagentTaskStore // optional: durable async completion ledger + usageCaps *usagecaps.Service + admission *orchestration.ChildRunAdmission + sweeperOnce sync.Once + sweeperStop chan struct{} + sweeperDone chan struct{} + sweeperStarted bool + lifecycleMu sync.Mutex + lifecycleClosed bool + lifecycleWG sync.WaitGroup + closeOnce sync.Once + closeDone chan struct{} + now func() time.Time + // Defaults used only when a task was created without caller-specific context. contextWindow int maxTokens int } @@ -104,6 +136,31 @@ func NewSubagentManager( createTools func() *Registry, cfg SubagentConfig, ) *SubagentManager { + return NewSubagentManagerWithAdmission( + provider, + providerReg, + model, + msgBus, + createTools, + cfg, + orchestration.NewChildRunAdmission(32, 128), + ) +} + +// NewSubagentManagerWithAdmission creates a manager sharing the process-owned +// child-run admission controller with other child execution paths. +func NewSubagentManagerWithAdmission( + provider providers.Provider, + providerReg *providers.Registry, + model string, + msgBus *bus.MessageBus, + createTools func() *Registry, + cfg SubagentConfig, + admission *orchestration.ChildRunAdmission, +) *SubagentManager { + if admission == nil { + admission = orchestration.NewChildRunAdmission(32, 128) + } return &SubagentManager{ tasks: make(map[string]*SubagentTask), config: cfg, @@ -112,6 +169,11 @@ func NewSubagentManager( model: model, msgBus: msgBus, createTools: createTools, + admission: admission, + sweeperStop: make(chan struct{}), + sweeperDone: make(chan struct{}), + closeDone: make(chan struct{}), + now: time.Now, } } @@ -121,7 +183,7 @@ func (sm *SubagentManager) SetAnnounceQueue(q *AnnounceQueue) { sm.announceQueue = q } -// SetTaskStore sets the persistent store for subagent tasks (write-through, fire-and-forget). +// SetTaskStore sets the persistent store for task lifecycle and async retrieval. func (sm *SubagentManager) SetTaskStore(s store.SubagentTaskStore) { sm.taskStore = s } diff --git a/internal/tools/subagent_announce_backpressure_test.go b/internal/tools/subagent_announce_backpressure_test.go new file mode 100644 index 00000000..19937f0f --- /dev/null +++ b/internal/tools/subagent_announce_backpressure_test.go @@ -0,0 +1,55 @@ +package tools + +import ( + "context" + "testing" + "time" + + "github.com/google/uuid" + + "github.com/nextlevelbuilder/goclaw/internal/bus" +) + +func TestSubagentDirectAnnouncementDoesNotBlockOnFullInboundBus(t *testing.T) { + messageBus := bus.New() + for range 1000 { + messageBus.PublishInbound(bus.InboundMessage{Content: "fill"}) + } + manager := NewSubagentManager(nil, nil, "", messageBus, nil, SubagentConfig{}) + taskStore := newRecordingSubagentTaskStore() + manager.SetTaskStore(taskStore) + task := &SubagentTask{ + ID: "task-1", + Label: "probe", + Status: TaskStatusCompleted, + Result: "done", + CreatedAt: time.Now().Add(-time.Second).UnixMilli(), + OriginChannel: "test", + OriginChatID: "chat-1", + OriginTenantID: uuid.New(), + RootAgentID: uuid.New(), + RootAgentKey: "root", + OriginSessionKey: "session-1", + dbID: uuid.New(), + } + + done := make(chan struct{}) + go func() { + manager.announceTask(context.Background(), task, nil, 1, true) + close(done) + }() + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("subagent announcement blocked on a full inbound bus") + } + select { + case metadata := <-taskStore.metadata: + if metadata[asyncCompletionDeliveryKey] != asyncCompletionDeliveryMissed { + t.Fatalf("announcement metadata = %#v, want undelivered", metadata) + } + case <-time.After(time.Second): + t.Fatal("subagent missed announcement was not recorded after bus saturation") + } +} diff --git a/internal/tools/subagent_announce_key_test.go b/internal/tools/subagent_announce_key_test.go new file mode 100644 index 00000000..34b35650 --- /dev/null +++ b/internal/tools/subagent_announce_key_test.go @@ -0,0 +1,47 @@ +package tools + +import ( + "testing" + + "github.com/google/uuid" +) + +func TestSubagentAnnounceBatchKeyScopesRoutingAndAuthority(t *testing.T) { + base := SubagentTask{ + OriginTenantID: uuid.New(), + RootAgentID: uuid.New(), + RootAgentKey: "root", + OriginSessionKey: "session", + OriginChannel: "telegram", + OriginChatID: "chat", + OriginPeerKind: "group", + OriginLocalKey: "topic", + OriginUserID: "user", + OriginSenderID: "sender", + OriginRole: "operator", + } + want := subagentAnnounceBatchKey(&base) + + cases := map[string]func(*SubagentTask){ + "tenant": func(v *SubagentTask) { v.OriginTenantID = uuid.New() }, + "root id": func(v *SubagentTask) { v.RootAgentID = uuid.New() }, + "root key": func(v *SubagentTask) { v.RootAgentKey += "-other" }, + "session": func(v *SubagentTask) { v.OriginSessionKey += "-other" }, + "channel": func(v *SubagentTask) { v.OriginChannel += "-other" }, + "chat": func(v *SubagentTask) { v.OriginChatID += "-other" }, + "peer kind": func(v *SubagentTask) { v.OriginPeerKind += "-other" }, + "local key": func(v *SubagentTask) { v.OriginLocalKey += "-other" }, + "user": func(v *SubagentTask) { v.OriginUserID += "-other" }, + "sender": func(v *SubagentTask) { v.OriginSenderID += "-other" }, + "role": func(v *SubagentTask) { v.OriginRole += "-other" }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + got := base + mutate(&got) + if key := subagentAnnounceBatchKey(&got); key == want { + t.Fatalf("batch key did not change when %s changed", name) + } + }) + } +} diff --git a/internal/tools/subagent_config.go b/internal/tools/subagent_config.go index 1466114b..628b661e 100644 --- a/internal/tools/subagent_config.go +++ b/internal/tools/subagent_config.go @@ -1,15 +1,17 @@ package tools -import "fmt" +import ( + "fmt" +) -// DefaultSubagentConfig returns sensible defaults matching OpenClaw TS spec. -// TS sources: agent-limits.ts, sessions-spawn-tool.ts, subagent-registry.ts. +// DefaultSubagentConfig returns GoClaw's runtime defaults. Per-root admission is +// independent from the Standard/Lite process safety cap. func DefaultSubagentConfig() SubagentConfig { return SubagentConfig{ - MaxConcurrent: 8, // TS: DEFAULT_SUBAGENT_MAX_CONCURRENT = 8 - MaxSpawnDepth: 1, // TS: maxSpawnDepth ?? 1 - MaxChildrenPerAgent: 5, // TS: maxChildrenPerAgent ?? 5 - ArchiveAfterMinutes: 60, // TS: archiveAfterMinutes ?? 60 + MaxConcurrent: 20, + MaxSpawnDepth: 1, + MaxChildrenPerAgent: 5, + ArchiveAfterMinutes: 60, MaxRetries: 2, } } @@ -74,8 +76,8 @@ Your final response IS the deliverable — it will be forwarded to the user. ## Sub-Agent Spawning You CAN spawn your own sub-agents for parallel or complex work using the spawn tool. -Your sub-agents will announce their results back to you automatically (not to the main agent). -Coordinate their work and synthesize results before reporting back.` +Asynchronous descendants report to the root orchestrator with their direct-parent lineage. +Synchronous descendants return their result directly to you.` } else if task.Depth >= 2 { prompt += ` diff --git a/internal/tools/subagent_context_window_test.go b/internal/tools/subagent_context_window_test.go index 2f65ff2e..4392b1f2 100644 --- a/internal/tools/subagent_context_window_test.go +++ b/internal/tools/subagent_context_window_test.go @@ -6,6 +6,8 @@ import ( "strings" "testing" + "github.com/google/uuid" + "github.com/nextlevelbuilder/goclaw/internal/providers" "github.com/nextlevelbuilder/goclaw/internal/store" usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps" @@ -46,8 +48,11 @@ func TestSubagentSpawn_CapturesOriginContextWindow(t *testing.T) { // Calling agent configured at 128k/8192, propagated via ctx (as injectContext does). ctx := store.WithAgentContextWindow(context.Background(), 128_000) ctx = store.WithAgentMaxTokens(ctx, 8_192) + ctx = store.WithTenantID(ctx, uuid.New()) + ctx = store.WithAgentID(ctx, uuid.New()) + ctx = WithToolAgentKey(ctx, "parent") - _, _, err := manager.RunSync(ctx, "parent", 0, "task", "label", "", "chan", "chat") + _, _, _, err := manager.RunSync(ctx, "parent", 0, "task", "label", "", "chan", "chat") if err != nil { t.Fatalf("RunSync error: %v", err) } diff --git a/internal/tools/subagent_control.go b/internal/tools/subagent_control.go index 9a59c049..15d6fde2 100644 --- a/internal/tools/subagent_control.go +++ b/internal/tools/subagent_control.go @@ -7,37 +7,180 @@ import ( "fmt" "log/slog" "strings" + "sync" "time" "unicode/utf8" + + "github.com/google/uuid" + "github.com/nextlevelbuilder/goclaw/internal/bus" + "github.com/nextlevelbuilder/goclaw/internal/store" ) -// scheduleArchive removes a task after the archive TTL. -func (sm *SubagentManager) scheduleArchive(taskID string, after time.Duration) { - time.Sleep(after) +const ( + subagentArchiveSweepInterval = time.Minute + subagentArchiveBatchSize = 256 +) + +func (sm *SubagentManager) startArchiveSweeper() { + sm.lifecycleMu.Lock() + defer sm.lifecycleMu.Unlock() + if sm.lifecycleClosed { + return + } + sm.sweeperOnce.Do(func() { + sm.mu.Lock() + sm.sweeperStarted = true + sm.mu.Unlock() + go func() { + ticker := time.NewTicker(subagentArchiveSweepInterval) + defer ticker.Stop() + defer close(sm.sweeperDone) + for { + select { + case <-ticker.C: + sm.sweepArchivedTasks() + case <-sm.sweeperStop: + return + } + } + }() + }) +} + +func (sm *SubagentManager) sweepArchivedTasks() { + now := sm.now() + type persistentSweep struct { + scope TaskScope + ttl time.Duration + } + persistent := make(map[string]persistentSweep) + sm.mu.Lock() - defer sm.mu.Unlock() - if t, ok := sm.tasks[taskID]; ok && t.Status != TaskStatusRunning { - delete(sm.tasks, taskID) - slog.Debug("subagent archived", "id", taskID) + removed := 0 + for id, task := range sm.tasks { + if removed >= subagentArchiveBatchSize || !isTerminalTaskStatus(task.Status) || + task.CompletedAt == 0 || task.spawnConfig.ArchiveAfterMinutes <= 0 || + now.Before(time.UnixMilli(task.CompletedAt).Add(time.Duration(task.spawnConfig.ArchiveAfterMinutes)*time.Minute)) || + !taskExecutionDone(task) { + continue + } + delete(sm.tasks, id) + removed++ + scope := TaskScope{ + TenantID: task.OriginTenantID, RootAgentID: task.RootAgentID, RootAgentKey: task.RootAgentKey, + } + persistent[scope.TenantID.String()+":"+scope.RootAgentID.String()] = persistentSweep{ + scope: scope, ttl: time.Duration(task.spawnConfig.ArchiveAfterMinutes) * time.Minute, + } + } + sm.mu.Unlock() + + for _, sweep := range persistent { + if sm.taskStore == nil || sweep.scope.RootAgentID == uuid.Nil { + continue + } + ctx := store.WithTenantID(context.Background(), sweep.scope.TenantID) + if _, err := sm.taskStore.Archive(ctx, sweep.scope.RootAgentID, sweep.ttl, subagentArchiveBatchSize); err != nil { + slog.Warn("subagent archive sweep failed", "root_agent", sweep.scope.RootAgentKey, "error", err) + } } } -// GetTask returns a task by ID. -func (sm *SubagentManager) GetTask(id string) (*SubagentTask, bool) { +// Close stops the manager-owned archive sweeper and waits for accepted task +// lifecycle work. Gateway owners should use CloseContext so dependency teardown +// remains bounded. +func (sm *SubagentManager) Close() { + _ = sm.CloseContext(context.Background()) +} + +func (sm *SubagentManager) CloseContext(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() + } + sm.closeOnce.Do(func() { + sm.lifecycleMu.Lock() + sm.lifecycleClosed = true + sm.lifecycleMu.Unlock() + + sm.mu.RLock() + started := sm.sweeperStarted + sm.mu.RUnlock() + close(sm.sweeperStop) + go func() { + if started { + <-sm.sweeperDone + } + sm.lifecycleWG.Wait() + close(sm.closeDone) + }() + }) + if sm.announceQueue != nil { + if err := sm.announceQueue.CloseContext(ctx); err != nil { + return fmt.Errorf("%w: %v", ErrSubagentLifecycleDrainTimeout, err) + } + } + select { + case <-sm.closeDone: + return nil + case <-ctx.Done(): + return fmt.Errorf("%w: %v", ErrSubagentLifecycleDrainTimeout, ctx.Err()) + } +} + +// beginLifecycleOperation registers task ownership before admission or durable +// persistence begins. Gateway shutdown closes this gate, drains admission, and +// then waits for every registered operation before stores and message-bus +// dependencies are closed. +func (sm *SubagentManager) beginLifecycleOperation() (func(), bool) { + finishes, ok := sm.beginLifecycleOperations(1) + if !ok { + return nil, false + } + return finishes[0], true +} + +func (sm *SubagentManager) beginLifecycleOperations(count int) ([]func(), bool) { + if count < 1 { + return nil, false + } + sm.lifecycleMu.Lock() + defer sm.lifecycleMu.Unlock() + if sm.lifecycleClosed { + return nil, false + } + sm.lifecycleWG.Add(count) + finishes := make([]func(), count) + for i := range finishes { + var once sync.Once + finishes[i] = func() { + once.Do(sm.lifecycleWG.Done) + } + } + return finishes, true +} + +// GetTask returns an immutable snapshot when the caller owns the root tree. +func (sm *SubagentManager) GetTask(scope TaskScope, id string) (*SubagentTask, bool) { sm.mu.RLock() defer sm.mu.RUnlock() t, ok := sm.tasks[id] - return t, ok + if !ok || !taskMatchesScope(t, scope) { + return nil, false + } + return cloneSubagentTask(t), true } -// ListTasks returns all tasks, optionally filtered by parent. -func (sm *SubagentManager) ListTasks(parentID string) []*SubagentTask { +// ListTasks returns immutable snapshots from one tenant/root-agent tree. +func (sm *SubagentManager) ListTasks(scope TaskScope, parentTaskID string) []*SubagentTask { sm.mu.RLock() defer sm.mu.RUnlock() var result []*SubagentTask for _, t := range sm.tasks { - if parentID == "" || t.ParentID == parentID { - result = append(result, t) + if !taskMatchesScope(t, scope) { + continue + } + if parentTaskID == "" || t.ParentTaskID == parentTaskID { + result = append(result, cloneSubagentTask(t)) } } return result @@ -46,57 +189,107 @@ func (sm *SubagentManager) ListTasks(parentID string) []*SubagentTask { // CancelTask cancels a running task by ID. // Special IDs: "all" cancels all running tasks for any parent, // "last" cancels the most recently created running task. -func (sm *SubagentManager) CancelTask(id string) bool { +func (sm *SubagentManager) CancelTask(scope TaskScope, id string) bool { + finish, ok := sm.beginLifecycleOperation() + if !ok { + return false + } + transferred := false + defer func() { + if !transferred { + finish() + } + }() + sm.mu.Lock() - defer sm.mu.Unlock() + var cancelled []*SubagentTask if id == "all" { - cancelled := false for _, t := range sm.tasks { - if t.Status == TaskStatusRunning { + if taskMatchesScope(t, scope) && isActiveTaskStatus(t.Status) { sm.cancelTaskLocked(t) - cancelled = true + cancelled = append(cancelled, t) } } - return cancelled + sm.mu.Unlock() + if len(cancelled) > 0 { + transferred = true + go func() { + defer finish() + sm.persistCancelledTasks(cancelled) + }() + } + return len(cancelled) > 0 } if id == "last" { var latest *SubagentTask for _, t := range sm.tasks { - if t.Status == TaskStatusRunning { + if taskMatchesScope(t, scope) && isActiveTaskStatus(t.Status) { if latest == nil || t.CreatedAt > latest.CreatedAt { latest = t } } } if latest == nil { + sm.mu.Unlock() return false } sm.cancelTaskLocked(latest) + sm.mu.Unlock() + transferred = true + go func() { + defer finish() + sm.persistCancelledTasks([]*SubagentTask{latest}) + }() return true } t, ok := sm.tasks[id] - if !ok || t.Status != TaskStatusRunning { + if !ok || !taskMatchesScope(t, scope) || !isActiveTaskStatus(t.Status) { + sm.mu.Unlock() return false } sm.cancelTaskLocked(t) + sm.mu.Unlock() + transferred = true + go func() { + defer finish() + sm.persistCancelledTasks([]*SubagentTask{t}) + }() return true } // CancelTasksForParent cancels all running tasks for a specific parent. -func (sm *SubagentManager) CancelTasksForParent(parentID string) int { +func (sm *SubagentManager) CancelTasksForParent(scope TaskScope, parentTaskID string) int { + finish, ok := sm.beginLifecycleOperation() + if !ok { + return 0 + } + transferred := false + defer func() { + if !transferred { + finish() + } + }() + sm.mu.Lock() - defer sm.mu.Unlock() - count := 0 + var cancelled []*SubagentTask for _, t := range sm.tasks { - if t.ParentID == parentID && t.Status == TaskStatusRunning { + if taskMatchesScope(t, scope) && t.ParentTaskID == parentTaskID && isActiveTaskStatus(t.Status) { sm.cancelTaskLocked(t) - count++ + cancelled = append(cancelled, t) } } - return count + sm.mu.Unlock() + if len(cancelled) > 0 { + transferred = true + go func() { + defer finish() + sm.persistCancelledTasks(cancelled) + }() + } + return len(cancelled) } // cancelTaskLocked sets a task to cancelled and fires its context cancel. @@ -108,28 +301,53 @@ func (sm *SubagentManager) cancelTaskLocked(t *SubagentTask) { if t.cancelFunc != nil { t.cancelFunc() } + if t.admissionTicket != nil { + t.admissionTicket.Cancel() + } +} + +func (sm *SubagentManager) persistCancelledTasks(tasks []*SubagentTask) { + for _, task := range tasks { + ctx := store.WithTenantID(context.Background(), task.OriginTenantID) + sm.persistStatus(ctx, task, 0) + } } // Steer cancels a running subagent and restarts it with a new message. // Matching TS subagents-tool.ts steer action: cancel → settle → spawn replacement. func (sm *SubagentManager) Steer( ctx context.Context, + scope TaskScope, taskID, newMessage string, callback AsyncCallback, ) (string, error) { + finishes, ok := sm.beginLifecycleOperations(2) + if !ok { + return "", fmt.Errorf("subagent manager is closed") + } + finishSteer := finishes[0] + finishPersistence := finishes[1] + persistenceTransferred := false + defer finishSteer() + defer func() { + if !persistenceTransferred { + finishPersistence() + } + }() + sm.mu.Lock() t, ok := sm.tasks[taskID] - if !ok { + if !ok || !taskMatchesScope(t, scope) { sm.mu.Unlock() return "", fmt.Errorf("subagent %q not found", taskID) } - if t.Status != TaskStatusRunning { + if !isActiveTaskStatus(t.Status) { sm.mu.Unlock() return "", fmt.Errorf("subagent %q is not running (status=%s)", taskID, t.Status) } // Capture origin metadata before cancelling - parentID := t.ParentID + parentID := t.RootAgentKey depth := t.Depth - 1 // Spawn increments depth, so use original label := t.Label + " (steered)" model := t.Model @@ -140,6 +358,11 @@ func (sm *SubagentManager) Steer( // Cancel old task (suppress announce by marking cancelled before unlock) sm.cancelTaskLocked(t) sm.mu.Unlock() + persistenceTransferred = true + go func() { + defer finishPersistence() + sm.persistCancelledTasks([]*SubagentTask{t}) + }() // Brief settle period (matching TS 500ms settle) time.Sleep(500 * time.Millisecond) @@ -160,7 +383,7 @@ func (sm *SubagentManager) Steer( } // WaitForChildren blocks until all running tasks for parentID complete or timeout. -func (sm *SubagentManager) WaitForChildren(ctx context.Context, parentID string, timeoutSec int) ([]*SubagentTask, error) { +func (sm *SubagentManager) WaitForChildren(ctx context.Context, scope TaskScope, parentTaskID string, timeoutSec int) ([]*SubagentTask, error) { if timeoutSec <= 0 { timeoutSec = 300 } @@ -171,14 +394,14 @@ func (sm *SubagentManager) WaitForChildren(ctx context.Context, parentID string, for { select { case <-ctx.Done(): - return sm.ListTasks(parentID), ctx.Err() + return sm.ListTasks(scope, parentTaskID), ctx.Err() case <-deadline: - return sm.ListTasks(parentID), fmt.Errorf("timeout after %ds waiting for children", timeoutSec) + return sm.ListTasks(scope, parentTaskID), fmt.Errorf("timeout after %ds waiting for children", timeoutSec) case <-ticker.C: - tasks := sm.ListTasks(parentID) + tasks := sm.ListTasks(scope, parentTaskID) allDone := true for _, t := range tasks { - if t.Status == TaskStatusRunning { + if isActiveTaskStatus(t.Status) { allDone = false break } @@ -190,6 +413,47 @@ func (sm *SubagentManager) WaitForChildren(ctx context.Context, parentID string, } } +func taskMatchesScope(task *SubagentTask, scope TaskScope) bool { + return task != nil && + scope.TenantID != uuid.Nil && + scope.RootAgentID != uuid.Nil && + scope.RootAgentKey != "" && + task.OriginTenantID == scope.TenantID && + task.RootAgentID == scope.RootAgentID && + task.RootAgentKey == scope.RootAgentKey +} + +func isActiveTaskStatus(status string) bool { + return status == TaskStatusQueued || status == TaskStatusRunning || status == TaskStatusWaiting +} + +func isTerminalTaskStatus(status string) bool { + return status == TaskStatusCompleted || status == TaskStatusFailed || status == TaskStatusCancelled +} + +func taskExecutionDone(task *SubagentTask) bool { + if task.admissionTicket == nil { + return true + } + select { + case <-task.admissionTicket.Done(): + return true + default: + return false + } +} + +func cloneSubagentTask(task *SubagentTask) *SubagentTask { + if task == nil { + return nil + } + snapshot := *task + snapshot.Media = append([]bus.MediaFile(nil), task.Media...) + snapshot.cancelFunc = nil + snapshot.admissionTicket = nil + return &snapshot +} + func generateSubagentID() string { b := make([]byte, 8) rand.Read(b) diff --git a/internal/tools/subagent_exec.go b/internal/tools/subagent_exec.go index e443c704..a15efe5b 100644 --- a/internal/tools/subagent_exec.go +++ b/internal/tools/subagent_exec.go @@ -16,10 +16,15 @@ import ( usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps" ) -// runTask executes the subagent in a goroutine. -func (sm *SubagentManager) runTask(ctx context.Context, task *SubagentTask, callback AsyncCallback) { - iterations := sm.executeTask(ctx, task) - +// announceTask publishes terminal state after child-run execution capacity has +// been released. Slow bus consumers and callbacks must never hold admission. +func (sm *SubagentManager) announceTask( + ctx context.Context, + task *SubagentTask, + callback AsyncCallback, + iterations int, + terminalPersisted bool, +) { // Announce result to parent via bus (matching TS subagent-announce.ts pattern). // The announce goes through the parent agent's session so the agent can // reformulate the result for the user. @@ -27,15 +32,19 @@ func (sm *SubagentManager) runTask(ctx context.Context, task *SubagentTask, call elapsed := time.Since(time.UnixMilli(task.CreatedAt)) item := AnnounceQueueItem{ - SubagentID: task.ID, - Label: task.Label, - Status: task.Status, - Result: task.Result, - Media: task.Media, - Runtime: elapsed, - Iterations: iterations, - InputTokens: task.TotalInputTokens, - OutputTokens: task.TotalOutputTokens, + SubagentID: task.ID, + CompletionID: task.dbID, + DurablyPersisted: terminalPersisted, + ParentTaskID: task.ParentTaskID, + Depth: task.Depth, + Label: task.Label, + Status: task.Status, + Result: task.Result, + Media: task.Media, + Runtime: elapsed, + Iterations: iterations, + InputTokens: task.TotalInputTokens, + OutputTokens: task.TotalOutputTokens, } meta := AnnounceMetadata{ OriginChannel: task.OriginChannel, @@ -47,34 +56,40 @@ func (sm *SubagentManager) runTask(ctx context.Context, task *SubagentTask, call OriginRole: task.OriginRole, OriginSessionKey: task.OriginSessionKey, OriginTenantID: task.OriginTenantID, - ParentAgent: task.ParentID, + RootAgentID: task.RootAgentID, + ParentAgent: task.RootAgentKey, OriginTraceID: task.OriginTraceID.String(), OriginRootSpanID: task.OriginRootSpanID.String(), } if sm.announceQueue != nil { // Use batched announce queue (matching TS debounce pattern) - sessionKey := fmt.Sprintf("announce:%s:%s", task.ParentID, task.OriginChatID) + sessionKey := subagentAnnounceBatchKey(task) sm.announceQueue.Enqueue(sessionKey, item, meta) } else { // Direct publish (no batching) - roster := sm.RosterForParent(task.ParentID) + roster := sm.RosterForParent(TaskScope{ + TenantID: task.OriginTenantID, RootAgentID: task.RootAgentID, RootAgentKey: task.RootAgentKey, + }) announceContent := FormatBatchedAnnounce([]AnnounceQueueItem{item}, roster) announceMeta := map[string]string{ - MetaOriginChannel: task.OriginChannel, - MetaOriginPeerKind: task.OriginPeerKind, - MetaParentAgent: task.ParentID, - "subagent_id": task.ID, - MetaSubagentLabel: task.Label, - MetaSubagentStatus: task.Status, - MetaSubagentResult: task.Result, - MetaSubagentRuntime: fmt.Sprintf("%d", elapsed.Milliseconds()), - MetaSubagentIterations: fmt.Sprintf("%d", iterations), - MetaSubagentInputToks: fmt.Sprintf("%d", task.TotalInputTokens), - MetaSubagentOutputToks: fmt.Sprintf("%d", task.TotalOutputTokens), - MetaOriginTraceID: task.OriginTraceID.String(), - MetaOriginRootSpanID: task.OriginRootSpanID.String(), + MetaOriginChannel: task.OriginChannel, + MetaOriginPeerKind: task.OriginPeerKind, + MetaParentAgent: task.RootAgentKey, + MetaSubagentRootAgentID: task.RootAgentID.String(), + "subagent_id": task.ID, + MetaSubagentLabel: task.Label, + MetaSubagentStatus: task.Status, + MetaSubagentResult: task.Result, + MetaSubagentRuntime: fmt.Sprintf("%d", elapsed.Milliseconds()), + MetaSubagentIterations: fmt.Sprintf("%d", iterations), + MetaSubagentInputToks: fmt.Sprintf("%d", task.TotalInputTokens), + MetaSubagentOutputToks: fmt.Sprintf("%d", task.TotalOutputTokens), + MetaSubagentParentTask: task.ParentTaskID, + MetaSubagentDepth: fmt.Sprintf("%d", task.Depth), + MetaOriginTraceID: task.OriginTraceID.String(), + MetaOriginRootSpanID: task.OriginRootSpanID.String(), } if task.OriginLocalKey != "" { announceMeta[MetaOriginLocalKey] = task.OriginLocalKey @@ -91,7 +106,7 @@ func (sm *SubagentManager) runTask(ctx context.Context, task *SubagentTask, call if task.OriginUserID != "" { announceMeta[MetaOriginUserID] = task.OriginUserID } - sm.msgBus.PublishInbound(bus.InboundMessage{ + delivered := PublishAsyncCompletion(ctx, sm.msgBus, bus.InboundMessage{ Channel: "system", SenderID: fmt.Sprintf("subagent:%s", task.ID), ChatID: task.OriginChatID, @@ -101,6 +116,24 @@ func (sm *SubagentManager) runTask(ctx context.Context, task *SubagentTask, call Metadata: announceMeta, Media: task.Media, }) + if terminalPersisted { + sm.UpdateAnnouncementStatus(ctx, task.RootAgentID, task.dbID, delivered) + } else { + slog.Error("subagent.announce_without_durable_terminal", + "task_id", task.ID, + "completion_id", task.dbID, + "root_agent_id", task.RootAgentID, + "delivered", delivered, + ) + } + if !delivered { + slog.Warn("subagent.announce_deferred_to_ledger", + "task_id", task.ID, + "completion_id", task.dbID, + "root_agent_id", task.RootAgentID, + "reason", "inbound_bus_full", + ) + } } } @@ -112,6 +145,23 @@ func (sm *SubagentManager) runTask(ctx context.Context, task *SubagentTask, call } } +func subagentAnnounceBatchKey(task *SubagentTask) string { + return strings.Join([]string{ + "announce", + task.OriginTenantID.String(), + task.RootAgentID.String(), + task.RootAgentKey, + task.OriginSessionKey, + task.OriginChannel, + task.OriginChatID, + task.OriginPeerKind, + task.OriginLocalKey, + task.OriginUserID, + task.OriginSenderID, + task.OriginRole, + }, "\x00") +} + // executeTask runs the LLM tool loop for a subagent. Returns iteration count. func (sm *SubagentManager) executeTask(ctx context.Context, task *SubagentTask) int { // Tracing: generate a root span ID for this subagent execution. @@ -120,19 +170,15 @@ func (sm *SubagentManager) executeTask(ctx context.Context, task *SubagentTask) subRootSpanID := store.GenNewID() taskStart := time.Now().UTC() - // Use a detached context for tracing so spans are emitted even if parent ctx is cancelled. - // We copy tracing values but remove the cancellation chain. - traceCtx := context.Background() - if collector := tracing.CollectorFromContext(ctx); collector != nil { - traceCtx = tracing.WithCollector(traceCtx, collector) - traceCtx = tracing.WithTraceID(traceCtx, tracing.TraceIDFromContext(ctx)) - // Keep original parent_span_id (parent agent's root span) for the subagent root span. - traceCtx = tracing.WithParentSpanID(traceCtx, tracing.ParentSpanIDFromContext(ctx)) - } + // Detach cancellation while preserving the delegation redactor and the + // original trace parent. Rebuilding from Background would silently drop the + // artifact confidentiality boundary on nested synchronous work. + traceCtx := context.WithoutCancel(ctx) // subCtx overrides parent_span_id so child spans nest under subRootSpanID. // traceCtx retains the original parent_span_id for the root subagent span. subTraceCtx := tracing.WithParentSpanID(traceCtx, subRootSpanID) + toolCtx := tracing.WithParentSpanID(ctx, subRootSpanID) var model string var finalContent string @@ -150,10 +196,6 @@ func (sm *SubagentManager) executeTask(ctx context.Context, task *SubagentTask) "trace_id", tracing.TraceIDFromContext(traceCtx), "status", task.Status, "iterations", iteration) - // Schedule auto-archive - if task.spawnConfig.ArchiveAfterMinutes > 0 { - go sm.scheduleArchive(task.ID, time.Duration(task.spawnConfig.ArchiveAfterMinutes)*time.Minute) - } }() if ctx.Err() != nil { @@ -166,6 +208,7 @@ func (sm *SubagentManager) executeTask(ctx context.Context, task *SubagentTask) // Build tools for subagent (no spawn/subagent tools to prevent recursion) toolsReg := sm.createTools() + toolsReg.Register(NewSpawnTool(sm, task.RootAgentKey, task.Depth)) sm.applyDenyList(toolsReg, task.Depth, task.spawnConfig) // Determine model (cascading priority): @@ -201,11 +244,15 @@ func (sm *SubagentManager) executeTask(ctx context.Context, task *SubagentTask) // Build subagent system prompt (matching TS buildSubagentSystemPrompt pattern). workspace := ToolWorkspaceFromCtx(ctx) - systemPrompt := sm.buildSubagentSystemPrompt(task, task.spawnConfig, workspace) + promptWorkspace := workspace + if IsDelegationArtifactRun(ctx) { + promptWorkspace = "outputs/" + } + systemPrompt := tracing.RedactText(ctx, sm.buildSubagentSystemPrompt(task, task.spawnConfig, promptWorkspace)) messages := []providers.Message{ {Role: "system", Content: systemPrompt}, - {Role: "user", Content: task.Task}, + {Role: "user", Content: tracing.RedactText(ctx, task.Task)}, } // Run LLM iteration loop (similar to agent loop but simplified) @@ -273,10 +320,9 @@ func (sm *SubagentManager) executeTask(ctx context.Context, task *SubagentTask) if err != nil { sm.mu.Lock() task.Status = TaskStatusFailed - task.Result = fmt.Sprintf("LLM error at iteration %d: %v", iteration, err) + task.Result = tracing.RedactText(ctx, fmt.Sprintf("LLM error at iteration %d: %v", iteration, err)) sm.mu.Unlock() slog.Warn("subagent LLM error", "id", task.ID, "iteration", iteration, "error", err) - go sm.persistStatus(ctx, task, iteration) return iteration } @@ -301,7 +347,8 @@ func (sm *SubagentManager) executeTask(ctx context.Context, task *SubagentTask) argsJSON, _ := json.Marshal(tc.Arguments) toolStart := time.Now().UTC() toolSpanID := sm.emitToolSpanStart(subTraceCtx, toolStart, tc.Name, tc.ID, string(argsJSON)) - result := toolsReg.Execute(ctx, tc.Name, tc.Arguments) + result := toolsReg.Execute(toolCtx, tc.Name, tc.Arguments) + result.ForLLM = tracing.RedactText(ctx, result.ForLLM) sm.emitToolSpanEnd(subTraceCtx, toolSpanID, toolStart, result.ForLLM, result.IsError) // Capture media file paths from tool results (e.g. image generation). @@ -327,19 +374,19 @@ func (sm *SubagentManager) executeTask(ctx context.Context, task *SubagentTask) } sm.mu.Lock() - if finalContent == "" { - finalContent = "Task completed but no final response was generated." + if task.Status != TaskStatusCancelled { + if finalContent == "" { + finalContent = "Task completed but no final response was generated." + } + finalContent = tracing.RedactText(ctx, finalContent) + task.Status = TaskStatusCompleted + task.Result = finalContent + task.Media = mediaFiles } - task.Status = TaskStatusCompleted - task.Result = finalContent - task.Media = mediaFiles sm.mu.Unlock() slog.Info("subagent completed", "id", task.ID, "iterations", iteration) - // Persist final status to DB (fire-and-forget). - go sm.persistStatus(ctx, task, iteration) - return iteration } diff --git a/internal/tools/subagent_persist.go b/internal/tools/subagent_persist.go index c50008da..30782a90 100644 --- a/internal/tools/subagent_persist.go +++ b/internal/tools/subagent_persist.go @@ -2,6 +2,7 @@ package tools import ( "context" + "fmt" "log/slog" "github.com/google/uuid" @@ -9,6 +10,17 @@ import ( "github.com/nextlevelbuilder/goclaw/internal/store" ) +const ( + asyncCompletionKindKey = "completion_kind" + asyncCompletionKindSubagent = "subagent" + asyncCompletionKindDelegate = "delegate" + asyncCompletionRuntimeIDKey = "runtime_task_id" + asyncCompletionDeliveryKey = "announcement_status" + asyncCompletionDeliveryPending = "pending" + asyncCompletionDeliveryDone = "delivered" + asyncCompletionDeliveryMissed = "undelivered" +) + // detachedCtx creates a context that won't be cancelled but preserves tenant ID. // Used for fire-and-forget DB writes that must succeed even after the parent ctx is cancelled. func detachedCtx(ctx context.Context) context.Context { @@ -19,10 +31,12 @@ func detachedCtx(ctx context.Context) context.Context { return bg } -// persistCreate writes a new subagent task to the DB (fire-and-forget). -func (sm *SubagentManager) persistCreate(ctx context.Context, task *SubagentTask) { +// persistCreate writes the accepted queued task before its admission ticket is +// activated. When a store is configured, execution is not accepted unless this +// write succeeds so every returned completion ID remains retrievable. +func (sm *SubagentManager) persistCreate(ctx context.Context, task *SubagentTask) error { if sm.taskStore == nil { - return + return nil } dbCtx := detachedCtx(ctx) @@ -52,10 +66,24 @@ func (sm *SubagentManager) persistCreate(ctx context.Context, task *SubagentTask originUserID = &task.OriginUserID } + var spawnedBy *uuid.UUID + if task.ParentTaskID != "" { + sm.mu.RLock() + if parent := sm.tasks[task.ParentTaskID]; parent != nil && parent.dbID != uuid.Nil && + taskMatchesScope(parent, TaskScope{ + TenantID: task.OriginTenantID, RootAgentID: task.RootAgentID, RootAgentKey: task.RootAgentKey, + }) { + parentID := parent.dbID + spawnedBy = &parentID + } + sm.mu.RUnlock() + } + data := &store.SubagentTaskData{ BaseModel: store.BaseModel{ID: task.dbID}, TenantID: task.OriginTenantID, - ParentAgentKey: task.ParentID, + RootAgentID: task.RootAgentID, + ParentAgentKey: task.RootAgentKey, SessionKey: sessionKey, Subject: task.Label, Description: task.Task, @@ -67,31 +95,129 @@ func (sm *SubagentManager) persistCreate(ctx context.Context, task *SubagentTask OriginChatID: originChatID, OriginPeerKind: originPeerKind, OriginUserID: originUserID, + SpawnedBy: spawnedBy, + Metadata: map[string]any{ + "root_agent_id": task.RootAgentID.String(), + "parent_task_id": task.ParentTaskID, + "depth": task.Depth, + asyncCompletionKindKey: asyncCompletionKindSubagent, + asyncCompletionRuntimeIDKey: task.ID, + asyncCompletionDeliveryKey: asyncCompletionDeliveryPending, + }, } - if err := sm.taskStore.Create(dbCtx, data); err != nil { - slog.Warn("subagent_persist: create failed", "id", task.ID, "error", err) + if err := retryAsyncPersistence(dbCtx, func(attemptCtx context.Context) error { + return sm.taskStore.Create(attemptCtx, data) + }); err != nil { + return fmt.Errorf("persist accepted subagent %s: %w", task.ID, err) } + return nil } -// persistStatus updates status, result, iterations, and token counts in the DB (fire-and-forget). -func (sm *SubagentManager) persistStatus(ctx context.Context, task *SubagentTask, iterations int) { +// persistStatus synchronously updates status, result, iterations, and token +// counts. Callers that announce a terminal result must check the returned error +// before recording an announcement delivery state. +func (sm *SubagentManager) persistStatus(ctx context.Context, task *SubagentTask, iterations int) error { if sm.taskStore == nil || task.dbID == uuid.Nil { - return + return nil } dbCtx := detachedCtx(ctx) + sm.mu.RLock() + snapshot := cloneSubagentTask(task) + sm.mu.RUnlock() + var result *string - if task.Result != "" { - result = &task.Result + if snapshot.Result != "" { + result = &snapshot.Result } - if err := sm.taskStore.UpdateStatus( - dbCtx, task.dbID, - task.Status, result, iterations, - task.TotalInputTokens, task.TotalOutputTokens, - ); err != nil { + if media := completionMediaDescriptors( + snapshot.Media, + snapshot.Workspace, + snapshot.MediaPathPrefix, + ); len(media) > 0 { + if err := retryTerminalPersistence(dbCtx, func(attemptCtx context.Context) error { + return sm.taskStore.UpdateMetadata(attemptCtx, snapshot.RootAgentID, snapshot.dbID, map[string]any{ + asyncCompletionMediaKey: media, + }) + }); err != nil { + slog.Warn("subagent_persist: update completion media failed", "id", task.ID, "error", err) + return err + } + } + + if err := retryTerminalPersistence(dbCtx, func(attemptCtx context.Context) error { + return sm.taskStore.UpdateStatus( + attemptCtx, snapshot.RootAgentID, snapshot.dbID, + snapshot.Status, result, iterations, + snapshot.TotalInputTokens, snapshot.TotalOutputTokens, + ) + }); err != nil { slog.Warn("subagent_persist: update status failed", "id", task.ID, "error", err) + return err + } + return nil +} + +// UpdateAnnouncementStatus records whether the automatic parent resume message +// was delivered. A missed delivery is recoverable through spawn(action="get"). +func (sm *SubagentManager) UpdateAnnouncementStatus( + ctx context.Context, + rootAgentID, completionID uuid.UUID, + delivered bool, +) { + if sm.taskStore == nil || rootAgentID == uuid.Nil || completionID == uuid.Nil { + return + } + status := asyncCompletionDeliveryMissed + if delivered { + status = asyncCompletionDeliveryDone + } + dbCtx := detachedCtx(ctx) + if err := retryAsyncPersistence(dbCtx, func(attemptCtx context.Context) error { + return sm.taskStore.UpdateMetadata(attemptCtx, rootAgentID, completionID, map[string]any{ + asyncCompletionDeliveryKey: status, + }) + }); err != nil { + slog.Warn("subagent_persist: update announcement status failed", + "completion_id", completionID, + "root_agent_id", rootAgentID, + "error", err, + ) } } + +// GetPersistedTask loads one durable self-clone result within the caller's +// tenant and root-agent scope. Delegate rows intentionally do not satisfy this +// contract. +func (sm *SubagentManager) GetPersistedTask( + ctx context.Context, + scope TaskScope, + completionID uuid.UUID, +) (*store.SubagentTaskData, error) { + if sm.taskStore == nil { + return nil, fmt.Errorf("durable subagent task tracking is unavailable") + } + if scope.TenantID == uuid.Nil || scope.RootAgentID == uuid.Nil || completionID == uuid.Nil { + return nil, fmt.Errorf("subagent completion lookup requires tenant, root agent, and completion ID") + } + dbCtx := store.WithTenantID(context.WithoutCancel(ctx), scope.TenantID) + task, err := sm.taskStore.Get(dbCtx, scope.RootAgentID, completionID) + if err != nil || task == nil { + return task, err + } + if completionKind(task.Metadata) == asyncCompletionKindDelegate { + return nil, nil + } + return task, nil +} + +func completionKind(metadata map[string]any) string { + if metadata == nil { + return "" + } + value, _ := metadata[asyncCompletionKindKey].(string) + return value +} diff --git a/internal/tools/subagent_roster.go b/internal/tools/subagent_roster.go index e73d839e..9d96a1ae 100644 --- a/internal/tools/subagent_roster.go +++ b/internal/tools/subagent_roster.go @@ -15,27 +15,36 @@ type SubagentRosterEntry struct { // including per-agent config limits for deterministic LLM context. type SubagentRoster struct { Entries []SubagentRosterEntry - Total int // total tasks for this parent + Active int // currently running tasks for this parent + Total int // retained tasks for this parent MaxPerAgent int // from spawnConfig.MaxChildrenPerAgent } -// RosterForParent returns the full roster of tasks for a parent. +// RosterForParent returns the full roster for one tenant/root-agent tree. // Sorted: completed/failed/cancelled first, then running (deterministic output). -func (sm *SubagentManager) RosterForParent(parentID string) SubagentRoster { +func (sm *SubagentManager) RosterForParent(scope TaskScope) SubagentRoster { sm.mu.RLock() defer sm.mu.RUnlock() var entries []SubagentRosterEntry - maxPerAgent := 0 + active := 0 + maxPerAgent := sm.config.MaxChildrenPerAgent + var latestCreatedAt int64 + var latestID string for _, t := range sm.tasks { - if t.ParentID != parentID { + if !taskMatchesScope(t, scope) { continue } entries = append(entries, SubagentRosterEntry{ Label: t.Label, Status: t.Status, }) - if maxPerAgent == 0 { + if isActiveTaskStatus(t.Status) { + active++ + } + if t.CreatedAt > latestCreatedAt || (t.CreatedAt == latestCreatedAt && t.ID > latestID) { + latestCreatedAt = t.CreatedAt + latestID = t.ID maxPerAgent = t.spawnConfig.MaxChildrenPerAgent } } @@ -56,6 +65,7 @@ func (sm *SubagentManager) RosterForParent(parentID string) SubagentRoster { return SubagentRoster{ Entries: entries, + Active: active, Total: len(entries), MaxPerAgent: maxPerAgent, } diff --git a/internal/tools/subagent_roster_test.go b/internal/tools/subagent_roster_test.go new file mode 100644 index 00000000..95722c8e --- /dev/null +++ b/internal/tools/subagent_roster_test.go @@ -0,0 +1,64 @@ +package tools + +import ( + "strings" + "testing" + + "github.com/google/uuid" +) + +func TestRosterForParentSeparatesActiveFromRetainedTasks(t *testing.T) { + tenantID := uuid.New() + rootID := uuid.New() + manager := NewSubagentManager(nil, nil, "", nil, nil, SubagentConfig{ + MaxChildrenPerAgent: 5, + }) + manager.tasks = map[string]*SubagentTask{ + "completed": { + ID: "completed", + ParentID: "parent", + RootAgentID: rootID, + RootAgentKey: "parent", + OriginTenantID: tenantID, + Label: "completed task", + Status: TaskStatusCompleted, + CreatedAt: 1, + spawnConfig: SubagentConfig{MaxChildrenPerAgent: 3}, + }, + "running": { + ID: "running", + ParentID: "parent", + RootAgentID: rootID, + RootAgentKey: "parent", + OriginTenantID: tenantID, + Label: "running task", + Status: TaskStatusRunning, + CreatedAt: 2, + spawnConfig: SubagentConfig{MaxChildrenPerAgent: 7}, + }, + "failed": { + ID: "failed", + ParentID: "parent", + RootAgentID: rootID, + RootAgentKey: "parent", + OriginTenantID: tenantID, + Label: "failed task", + Status: TaskStatusFailed, + CreatedAt: 3, + spawnConfig: SubagentConfig{MaxChildrenPerAgent: 9}, + }, + } + + roster := manager.RosterForParent(TaskScope{TenantID: tenantID, RootAgentID: rootID, RootAgentKey: "parent"}) + if roster.Active != 1 || roster.Total != 3 || roster.MaxPerAgent != 9 { + t.Fatalf("roster counts = %#v, want 1 active / 3 retained / max 9", roster) + } + + instruction := BuildReplyInstruction(roster) + if !strings.Contains(instruction, "1 active / 9 max; 3 retained total") { + t.Fatalf("instruction missing active/retained counts:\n%s", instruction) + } + if !strings.Contains(instruction, "1 subagent(s) still running") { + t.Fatalf("instruction missing running state:\n%s", instruction) + } +} diff --git a/internal/tools/subagent_spawn.go b/internal/tools/subagent_spawn.go index 3d275732..cdb86eba 100644 --- a/internal/tools/subagent_spawn.go +++ b/internal/tools/subagent_spawn.go @@ -6,14 +6,134 @@ import ( "log/slog" "time" + "github.com/google/uuid" + "github.com/nextlevelbuilder/goclaw/internal/bus" + orchestration "github.com/nextlevelbuilder/goclaw/internal/childrun" "github.com/nextlevelbuilder/goclaw/internal/edition" "github.com/nextlevelbuilder/goclaw/internal/store" "github.com/nextlevelbuilder/goclaw/internal/tracing" ) -// Spawn creates a new subagent task that runs asynchronously. -// Returns immediately with a status message. The subagent runs in a goroutine. -// modelOverride optionally overrides the LLM model for this subagent (matching TS sessions-spawn-tool.ts). +// SpawnReceipt identifies both the in-memory runtime task and its durable +// completion row. CompletionID is empty only when no task store is configured. +type SpawnReceipt struct { + TaskID string + CompletionID uuid.UUID + Message string +} + +// SpawnWithReceipt creates an asynchronous self-clone and returns identifiers +// that remain useful if the automatic parent announcement cannot be delivered. +func (sm *SubagentManager) SpawnWithReceipt( + ctx context.Context, + parentID string, + depth int, + task, label, modelOverride string, + channel, chatID, peerKind string, + callback AsyncCallback, +) (SpawnReceipt, error) { + finishLifecycle, ok := sm.beginLifecycleOperation() + if !ok { + return SpawnReceipt{}, fmt.Errorf("subagent manager is closed") + } + lifecycleTransferred := false + defer func() { + if !lifecycleTransferred { + finishLifecycle() + } + }() + + if err := validateDelegationChildRunMode(ctx, "spawn", "async"); err != nil { + return SpawnReceipt{}, err + } + cfg := sm.effectiveSpawnConfig(ctx) + depth = subagentDepthFromContext(ctx, depth) + if depth >= cfg.MaxSpawnDepth { + return SpawnReceipt{}, fmt.Errorf("spawn depth limit reached (%d/%d)", depth, cfg.MaxSpawnDepth) + } + + scope := subagentScopeFromContext(ctx) + if scope.RootAgentKey == "" { + scope.RootAgentKey = parentID + } + if scope.TenantID == uuid.Nil || scope.RootAgentID == uuid.Nil || scope.RootAgentKey == "" { + return SpawnReceipt{}, fmt.Errorf("spawn requires tenant and root-agent context") + } + parentTaskID := subagentTaskIDFromContext(ctx) + admissionParentID := subagentAdmissionParentID(parentTaskID, scope) + subTask := newSubagentTask(ctx, scope, parentTaskID, depth, task, label, modelOverride, channel, chatID, peerKind, cfg) + admissionParentID, admissionDepth := childRunContinuationLineage( + ctx, + admissionParentID, + subTask.Depth, + ) + // Detach from parent's cancellation chain so subagent survives after parent run completes. + // WithoutCancel preserves all context values (agent ID, workspace, trace info, etc.) + // but parent Done() no longer propagates. Manual cancel via taskCancel() still works. + detached := context.WithoutCancel(ctx) + taskCtx, taskCancel := context.WithCancel(detached) + subTask.cancelFunc = taskCancel + + if sm.taskStore != nil { + subTask.dbID = store.GenNewID() + } + + var ticket *orchestration.ChildRunTicket + var err error + var iterations int + ticket, err = sm.admission.Enqueue(taskCtx, orchestration.ChildRunConstraints{ + TenantID: scope.TenantID, + RootAgentID: scope.RootAgentID, + RootLimit: cfg.MaxConcurrent, + TaskID: subTask.ID, + ParentTaskID: admissionParentID, + ParentFanout: cfg.MaxChildrenPerAgent, + Depth: admissionDepth, + }, func(runCtx context.Context, lease *orchestration.ChildRunLease) { + sm.markTaskRunning(subTask) + iterations = sm.executeTask(withSubagentExecution(runCtx, scope, subTask.ID, subTask.Depth, lease), subTask) + lease.Release() + }) + if err != nil { + taskCancel() + return SpawnReceipt{}, err + } + subTask.admissionTicket = ticket + + if _, err := sm.acceptTask(taskCtx, subTask); err != nil { + ticket.Cancel() + taskCancel() + return SpawnReceipt{}, err + } + if err := ticket.Activate(); err != nil { + sm.rollbackRejectedTask(taskCtx, subTask, err) + taskCancel() + return SpawnReceipt{}, err + } + announceCtx := context.WithoutCancel(taskCtx) + completion := func() { + defer finishLifecycle() + <-ticket.Done() + sm.finishTicketTask(announceCtx, subTask, ticket, iterations) + terminalPersisted := sm.persistStatus(announceCtx, subTask, iterations) == nil + taskCancel() + sm.announceTask(announceCtx, subTask, callback, iterations, terminalPersisted) + } + lifecycleTransferred = true + go completion() + + slog.Info("subagent queued", "id", subTask.ID, "parent_task", parentTaskID, "depth", subTask.Depth, "label", subTask.Label) + + message := fmt.Sprintf("Spawned subagent '%s' (id=%s, depth=%d) for task: %s", + subTask.Label, subTask.ID, subTask.Depth, truncate(task, 100)) + return SpawnReceipt{ + TaskID: subTask.ID, + CompletionID: subTask.dbID, + Message: message, + }, nil +} + +// Spawn preserves the existing manager API for internal control flows. func (sm *SubagentManager) Spawn( ctx context.Context, parentID string, @@ -22,61 +142,160 @@ func (sm *SubagentManager) Spawn( channel, chatID, peerKind string, callback AsyncCallback, ) (string, error) { - cfg := sm.effectiveConfig(ctx) + receipt, err := sm.SpawnWithReceipt( + ctx, parentID, depth, task, label, modelOverride, + channel, chatID, peerKind, callback, + ) + return receipt.Message, err +} - // Apply edition ceilings (Lite edition enforces lower limits). - ed := edition.Current() - if ed.MaxSubagentConcurrent > 0 && cfg.MaxConcurrent > ed.MaxSubagentConcurrent { - cfg.MaxConcurrent = ed.MaxSubagentConcurrent +// RunSync executes a subagent task synchronously, blocking until completion. +func (sm *SubagentManager) RunSync( + ctx context.Context, + parentID string, + depth int, + task, label, modelOverride string, + channel, chatID string, +) (string, []bus.MediaFile, int, error) { + finishLifecycle, ok := sm.beginLifecycleOperation() + if !ok { + return "", nil, 0, fmt.Errorf("subagent manager is closed") } + defer finishLifecycle() + + cfg := sm.effectiveSpawnConfig(ctx) + depth = subagentDepthFromContext(ctx, depth) + if depth >= cfg.MaxSpawnDepth { + return "", nil, 0, fmt.Errorf("spawn depth limit reached (%d/%d)", depth, cfg.MaxSpawnDepth) + } + + scope := subagentScopeFromContext(ctx) + if scope.RootAgentKey == "" { + scope.RootAgentKey = parentID + } + if scope.TenantID == uuid.Nil || scope.RootAgentID == uuid.Nil || scope.RootAgentKey == "" { + return "", nil, 0, fmt.Errorf("spawn requires tenant and root-agent context") + } + parentTaskID := subagentTaskIDFromContext(ctx) + admissionParentID := subagentAdmissionParentID(parentTaskID, scope) + subTask := newSubagentTask(ctx, scope, parentTaskID, depth, task, label, modelOverride, channel, chatID, "", cfg) + admissionParentID, admissionDepth := childRunContinuationLineage( + ctx, + admissionParentID, + subTask.Depth, + ) + if sm.taskStore != nil { + subTask.dbID = store.GenNewID() + } + + taskCtx, taskCancel := context.WithCancel(ctx) + subTask.cancelFunc = taskCancel + constraints := orchestration.ChildRunConstraints{ + TenantID: scope.TenantID, + RootAgentID: scope.RootAgentID, + RootLimit: cfg.MaxConcurrent, + TaskID: subTask.ID, + ParentTaskID: admissionParentID, + ParentFanout: cfg.MaxChildrenPerAgent, + Depth: admissionDepth, + } + var iterations int + var acceptErr error + acceptedBeforeActivate := false + run := func(runCtx context.Context, lease *orchestration.ChildRunLease) { + if !acceptedBeforeActivate { + if _, acceptErr = sm.acceptTask(runCtx, subTask); acceptErr != nil { + lease.Release() + return + } + } + sm.markTaskRunning(subTask) + iterations = sm.executeTask(withSubagentExecution(runCtx, scope, subTask.ID, subTask.Depth, lease), subTask) + lease.Release() + sm.persistStatus(runCtx, subTask, iterations) + } + slog.Info("subagent sync queued", "id", subTask.ID, "parent_task", parentTaskID, "depth", subTask.Depth, "label", subTask.Label) + if parentLease := childRunLeaseFromContext(ctx); parentLease != nil { + sm.setTaskWaiting(parentTaskID, true) + err := parentLease.Continue(taskCtx, constraints, run) + sm.setTaskWaiting(parentTaskID, false) + if err != nil { + if sm.taskAccepted(subTask.ID) && !isTerminalTaskStatus(subTask.Status) { + sm.markTaskFailed(subTask, err) + sm.persistStatus(taskCtx, subTask, iterations) + } + taskCancel() + return "", nil, iterations, err + } + if acceptErr != nil { + taskCancel() + return "", nil, iterations, acceptErr + } + } else { + ticket, err := sm.admission.Enqueue(taskCtx, constraints, run) + if err != nil { + taskCancel() + return "", nil, 0, err + } + subTask.admissionTicket = ticket + if _, err := sm.acceptTask(taskCtx, subTask); err != nil { + ticket.Cancel() + taskCancel() + return "", nil, 0, err + } + acceptedBeforeActivate = true + if err := ticket.Activate(); err != nil { + sm.rollbackRejectedTask(taskCtx, subTask, err) + taskCancel() + return "", nil, 0, err + } + <-ticket.Done() + sm.finishTicketTask(taskCtx, subTask, ticket, iterations) + } + taskCancel() + + if subTask.Status == TaskStatusFailed { + return subTask.Result, nil, iterations, fmt.Errorf("subagent failed: %s", subTask.Result) + } + + media := append([]bus.MediaFile(nil), subTask.Media...) + return subTask.Result, media, iterations, nil +} + +// effectiveSpawnConfig applies per-agent overrides and edition ceilings once so +// async admission, sync admission, and pipeline batching use the same limits. +func (sm *SubagentManager) effectiveSpawnConfig(ctx context.Context) SubagentConfig { + cfg := sm.effectiveConfig(ctx) + ed := edition.Current() if ed.MaxSubagentDepth > 0 && cfg.MaxSpawnDepth > ed.MaxSubagentDepth { cfg.MaxSpawnDepth = ed.MaxSubagentDepth } + return cfg +} - sm.mu.Lock() - - // Check depth limit - if depth >= cfg.MaxSpawnDepth { - sm.mu.Unlock() - return "", fmt.Errorf("spawn depth limit reached (%d/%d)", depth, cfg.MaxSpawnDepth) - } - - // Check concurrent limit (scoped per tenant for isolation). - tenantID := store.TenantIDFromContext(ctx) - running := 0 - for _, t := range sm.tasks { - if t.Status == TaskStatusRunning && t.OriginTenantID == tenantID { - running++ - } - } - if running >= cfg.MaxConcurrent { - sm.mu.Unlock() - return "", fmt.Errorf("max concurrent subagents reached (%d/%d)", running, cfg.MaxConcurrent) - } - - // Check per-parent children limit - childCount := 0 - for _, t := range sm.tasks { - if t.ParentID == parentID { - childCount++ - } - } - if childCount >= cfg.MaxChildrenPerAgent { - sm.mu.Unlock() - return "", fmt.Errorf("max children per agent reached (%d/%d)", childCount, cfg.MaxChildrenPerAgent) - } - - id := generateSubagentID() +func newSubagentTask( + ctx context.Context, + scope TaskScope, + parentTaskID string, + depth int, + task, label, modelOverride string, + channel, chatID, peerKind string, + cfg SubagentConfig, +) *SubagentTask { if label == "" { label = truncate(task, 50) } - - subTask := &SubagentTask{ - ID: id, - ParentID: parentID, + mediaPathPrefix := "" + if IsDelegationArtifactRun(ctx) { + mediaPathPrefix = "outputs" + } + return &SubagentTask{ + ID: generateSubagentID(), + ParentID: scope.RootAgentKey, + ParentTaskID: parentTaskID, Task: task, Label: label, - Status: "running", + Status: TaskStatusQueued, Depth: depth + 1, Model: modelOverride, OriginChannel: channel, @@ -87,115 +306,119 @@ func (sm *SubagentManager) Spawn( OriginSenderID: store.SenderIDFromContext(ctx), OriginRole: store.RoleFromContext(ctx), OriginSessionKey: ToolSessionKeyFromCtx(ctx), - OriginAgentID: store.AgentIDFromContext(ctx), - OriginTenantID: store.TenantIDFromContext(ctx), + OriginAgentID: scope.RootAgentID, + OriginTenantID: scope.TenantID, + RootAgentID: scope.RootAgentID, + RootAgentKey: scope.RootAgentKey, OriginTraceID: tracing.TraceIDFromContext(ctx), OriginRootSpanID: tracing.ParentSpanIDFromContext(ctx), OriginContextWindow: store.AgentContextWindowFromContext(ctx), OriginMaxTokens: store.AgentMaxTokensFromContext(ctx), + Workspace: ToolWorkspaceFromCtx(ctx), + MediaPathPrefix: mediaPathPrefix, CreatedAt: time.Now().UnixMilli(), spawnConfig: cfg, } - // Detach from parent's cancellation chain so subagent survives after parent run completes. - // WithoutCancel preserves all context values (agent ID, workspace, trace info, etc.) - // but parent Done() no longer propagates. Manual cancel via taskCancel() still works. - detached := context.WithoutCancel(ctx) - taskCtx, taskCancel := context.WithCancel(detached) - subTask.cancelFunc = taskCancel - - // Assign DB UUID inside lock to avoid race with runTask goroutine. - if sm.taskStore != nil { - subTask.dbID = store.GenNewID() - } - - sm.tasks[id] = subTask - sm.mu.Unlock() - - slog.Info("subagent spawned", "id", id, "parent", parentID, "depth", subTask.Depth, "label", label) - - // Persist to DB (fire-and-forget). - if sm.taskStore != nil { - go sm.persistCreate(taskCtx, subTask) - } - - go sm.runTask(taskCtx, subTask, callback) - - return fmt.Sprintf("Spawned subagent '%s' (id=%s, depth=%d) for task: %s", - label, id, subTask.Depth, truncate(task, 100)), nil } -// RunSync executes a subagent task synchronously, blocking until completion. -func (sm *SubagentManager) RunSync( - ctx context.Context, - parentID string, - depth int, - task, label, modelOverride string, - channel, chatID string, -) (string, int, error) { - cfg := sm.effectiveConfig(ctx) - - // Apply edition ceilings (same as Spawn). - ed := edition.Current() - if ed.MaxSubagentConcurrent > 0 && cfg.MaxConcurrent > ed.MaxSubagentConcurrent { - cfg.MaxConcurrent = ed.MaxSubagentConcurrent - } - if ed.MaxSubagentDepth > 0 && cfg.MaxSpawnDepth > ed.MaxSubagentDepth { - cfg.MaxSpawnDepth = ed.MaxSubagentDepth - } - +func (sm *SubagentManager) markTaskRunning(task *SubagentTask) { sm.mu.Lock() - - if depth >= cfg.MaxSpawnDepth { - sm.mu.Unlock() - return "", 0, fmt.Errorf("spawn depth limit reached (%d/%d)", depth, cfg.MaxSpawnDepth) + defer sm.mu.Unlock() + if task.Status == TaskStatusCancelled { + return } + task.Status = TaskStatusRunning +} - id := generateSubagentID() - if label == "" { - label = truncate(task, 50) +func (sm *SubagentManager) setTaskWaiting(taskID string, waiting bool) { + if taskID == "" { + return } + sm.mu.Lock() + defer sm.mu.Unlock() + task := sm.tasks[taskID] + if task == nil || isTerminalTaskStatus(task.Status) { + return + } + if waiting { + task.Status = TaskStatusWaiting + } else { + task.Status = TaskStatusRunning + } +} + +func (sm *SubagentManager) markTaskFailed(task *SubagentTask, err error) { + sm.mu.Lock() + defer sm.mu.Unlock() + if !isTerminalTaskStatus(task.Status) { + task.Status = TaskStatusFailed + task.Result = err.Error() + task.CompletedAt = time.Now().UnixMilli() + } +} - subTask := &SubagentTask{ - ID: id, - ParentID: parentID, - Task: task, - Label: label, - Status: "running", - Depth: depth + 1, - Model: modelOverride, - OriginChannel: channel, - OriginChatID: chatID, - OriginLocalKey: ToolLocalKeyFromCtx(ctx), - OriginUserID: store.UserIDFromContext(ctx), - OriginSenderID: store.SenderIDFromContext(ctx), - OriginRole: store.RoleFromContext(ctx), - OriginSessionKey: ToolSessionKeyFromCtx(ctx), - OriginAgentID: store.AgentIDFromContext(ctx), - OriginTenantID: store.TenantIDFromContext(ctx), - OriginTraceID: tracing.TraceIDFromContext(ctx), - OriginRootSpanID: tracing.ParentSpanIDFromContext(ctx), - OriginContextWindow: store.AgentContextWindowFromContext(ctx), - OriginMaxTokens: store.AgentMaxTokensFromContext(ctx), - CreatedAt: time.Now().UnixMilli(), - spawnConfig: cfg, - } - if sm.taskStore != nil { - subTask.dbID = store.GenNewID() - } - sm.tasks[id] = subTask +func (sm *SubagentManager) acceptTask(ctx context.Context, task *SubagentTask) (bool, error) { + sm.mu.Lock() + if _, exists := sm.tasks[task.ID]; exists { + sm.mu.Unlock() + return false, fmt.Errorf("subagent task ID collision: %s", task.ID) + } + sm.tasks[task.ID] = task sm.mu.Unlock() + if err := sm.persistCreate(ctx, task); err != nil { + sm.mu.Lock() + if sm.tasks[task.ID] == task { + delete(sm.tasks, task.ID) + } + sm.mu.Unlock() + return false, err + } + if task.spawnConfig.ArchiveAfterMinutes > 0 { + sm.startArchiveSweeper() + } + return true, nil +} - slog.Info("subagent sync started", "id", id, "parent", parentID, "depth", subTask.Depth, "label", label) +func (sm *SubagentManager) taskAccepted(taskID string) bool { + sm.mu.RLock() + defer sm.mu.RUnlock() + _, exists := sm.tasks[taskID] + return exists +} - if sm.taskStore != nil { - sm.persistCreate(ctx, subTask) +func (sm *SubagentManager) rollbackRejectedTask(ctx context.Context, task *SubagentTask, err error) { + sm.markTaskFailed(task, err) + sm.persistStatus(ctx, task, 0) + sm.mu.Lock() + if sm.tasks[task.ID] == task { + delete(sm.tasks, task.ID) } + sm.mu.Unlock() +} - iterations := sm.executeTask(ctx, subTask) +func subagentAdmissionParentID(parentTaskID string, scope TaskScope) string { + if parentTaskID != "" { + return parentTaskID + } + return "root:" + scope.RootAgentID.String() +} - if subTask.Status == TaskStatusFailed { - return subTask.Result, iterations, fmt.Errorf("subagent failed: %s", subTask.Result) +func (sm *SubagentManager) finishTicketTask( + ctx context.Context, + task *SubagentTask, + ticket *orchestration.ChildRunTicket, + iterations int, +) { + sm.mu.RLock() + terminal := isTerminalTaskStatus(task.Status) + sm.mu.RUnlock() + if terminal { + return } - - return subTask.Result, iterations, nil + err := ticket.Err() + if err == nil { + err = fmt.Errorf("child run ended without terminal task state") + } + sm.markTaskFailed(task, err) + sm.persistStatus(ctx, task, iterations) } diff --git a/internal/tools/subagent_spawn_test.go b/internal/tools/subagent_spawn_test.go index f816be7a..f4be3a13 100644 --- a/internal/tools/subagent_spawn_test.go +++ b/internal/tools/subagent_spawn_test.go @@ -2,22 +2,149 @@ package tools import ( "context" + "os" + "path/filepath" + "strings" "testing" "github.com/google/uuid" + "github.com/nextlevelbuilder/goclaw/internal/bus" + "github.com/nextlevelbuilder/goclaw/internal/config" "github.com/nextlevelbuilder/goclaw/internal/providers" "github.com/nextlevelbuilder/goclaw/internal/store" + "github.com/nextlevelbuilder/goclaw/internal/tracing" ) type recordingSubagentProvider struct { - model string + model string + systemPrompt string + response string + calls int } func (p *recordingSubagentProvider) Name() string { return "recording" } func (p *recordingSubagentProvider) DefaultModel() string { return "provider-default" } func (p *recordingSubagentProvider) Chat(_ context.Context, req providers.ChatRequest) (*providers.ChatResponse, error) { + p.calls++ p.model = req.Model - return &providers.ChatResponse{Content: "done", FinishReason: "stop"}, nil + if len(req.Messages) > 0 { + p.systemPrompt = req.Messages[0].Content + } + content := p.response + if content == "" { + content = "done" + } + return &providers.ChatResponse{Content: content, FinishReason: "stop"}, nil +} + +func TestRunSyncKeepsParentAgentWorkspace(t *testing.T) { + provider := &recordingSubagentProvider{} + manager := NewSubagentManager(provider, nil, "manager-default", nil, NewRegistry, SubagentConfig{ + MaxConcurrent: 20, + MaxSpawnDepth: 1, + MaxChildrenPerAgent: 5, + }) + workspace := t.TempDir() + ctx := WithToolWorkspace(subagentTestContext("parent"), workspace) + + if _, _, _, err := manager.RunSync( + ctx, "parent", 0, "test workspace", "workspace", "", "test", "chat", + ); err != nil { + t.Fatalf("RunSync() error = %v", err) + } + if !strings.Contains(provider.systemPrompt, workspace) { + t.Fatalf("subagent prompt omitted parent workspace %q: %s", workspace, provider.systemPrompt) + } +} + +func TestDelegatedAgentStartsOwnSpawnTree(t *testing.T) { + provider := &recordingSubagentProvider{} + manager := NewSubagentManager(provider, nil, "manager-default", nil, NewRegistry, SubagentConfig{ + MaxConcurrent: 20, + MaxSpawnDepth: 1, + MaxChildrenPerAgent: 5, + }) + + tenantID := uuid.New() + agentA := uuid.New() + agentB := uuid.New() + ctx := store.WithTenantID(context.Background(), tenantID) + ctx = store.WithAgentID(ctx, agentA) + ctx = store.WithAgentKey(ctx, "agent-a") + ctx = WithToolAgentKey(ctx, "agent-a") + ctx = withSubagentExecution(ctx, TaskScope{ + TenantID: tenantID, RootAgentID: agentA, RootAgentKey: "agent-a", + }, "agent-a-leaf", 1, nil) + ctx = WithSubagentConfig(ctx, &config.SubagentsConfig{ + MaxConcurrent: 2, + MaxSpawnDepth: 4, + MaxChildrenPerAgent: 3, + }) + + ctx = withDelegatedAgentExecution(ctx, nil) + if inherited := SubagentConfigFromCtx(ctx); inherited != nil { + t.Fatalf("delegated agent inherited source config: %#v", inherited) + } + ctx = store.WithAgentID(ctx, agentB) + ctx = store.WithAgentKey(ctx, "agent-b") + ctx = store.WithAgentContextWindow(ctx, 200_000) + ctx = store.WithAgentMaxTokens(ctx, 32_000) + ctx = WithToolAgentKey(ctx, "agent-b") + ctx = WithSubagentConfig(ctx, &config.SubagentsConfig{ + MaxConcurrent: 20, + MaxSpawnDepth: 1, + MaxChildrenPerAgent: 5, + }) + + if _, _, _, err := manager.RunSync( + ctx, "agent-b", 0, "run as B child", "b-child", "", "delegate", "chat", + ); err != nil { + t.Fatalf("B spawn failed after delegation boundary: %v", err) + } + + scope := TaskScope{TenantID: tenantID, RootAgentID: agentB, RootAgentKey: "agent-b"} + tasks := manager.ListTasks(scope, "") + if len(tasks) != 1 { + t.Fatalf("B task count = %d, want 1", len(tasks)) + } + if task := tasks[0]; task.Depth != 1 || task.ParentTaskID != "" || + task.RootAgentID != agentB || task.RootAgentKey != "agent-b" { + t.Fatalf("B spawn tree task = %#v", task) + } +} + +func TestRunSyncRedactsDelegationArtifactWorkspace(t *testing.T) { + physicalOutputs := filepath.Join(t.TempDir(), "collaboration", "delegations", uuid.NewString(), "outputs") + if err := os.MkdirAll(physicalOutputs, 0750); err != nil { + t.Fatal(err) + } + provider := &recordingSubagentProvider{response: "result from " + physicalOutputs} + manager := NewSubagentManager(provider, nil, "manager-default", nil, NewRegistry, SubagentConfig{ + MaxConcurrent: 20, + MaxSpawnDepth: 2, + MaxChildrenPerAgent: 5, + }) + ctx := delegationArtifactTestContext() + ctx = store.WithAgentID(ctx, uuid.New()) + ctx = store.WithAgentContextWindow(ctx, 200_000) + ctx = store.WithAgentMaxTokens(ctx, 32_000) + ctx = WithToolWorkspace(ctx, physicalOutputs) + ctx = WithToolAgentKey(ctx, "parent") + ctx = tracing.WithTextRedactor(ctx, strings.NewReplacer(physicalOutputs, "outputs").Replace) + + result, _, _, err := manager.RunSync( + ctx, "parent", 0, "inspect outputs", "workspace", "", "test", "chat", + ) + if err != nil { + t.Fatalf("RunSync() error = %v", err) + } + if strings.Contains(provider.systemPrompt, physicalOutputs) || + !strings.Contains(provider.systemPrompt, "outputs/") { + t.Fatalf("artifact prompt leaked physical workspace: %s", provider.systemPrompt) + } + if strings.Contains(result, physicalOutputs) || !strings.Contains(result, "outputs") { + t.Fatalf("artifact result leaked physical workspace: %q", result) + } } func (p *recordingSubagentProvider) ChatStream(ctx context.Context, req providers.ChatRequest, _ func(providers.StreamChunk)) (*providers.ChatResponse, error) { return p.Chat(ctx, req) @@ -32,14 +159,14 @@ func TestRunSyncHonorsPerTaskModelOverride(t *testing.T) { Model: "configured-model", }) - ctx := store.WithTenantID(context.Background(), uuid.New()) + ctx := subagentTestContext("parent") ctx = WithParentModel(ctx, "parent-model") // The subagent's internal LLM call is agent-scoped; the guard requires a // budget in ctx (propagated from the calling agent via injectContext). ctx = store.WithAgentContextWindow(ctx, 200_000) ctx = store.WithAgentMaxTokens(ctx, 32_000) - result, _, err := manager.RunSync(ctx, "parent", 0, "test task", "test", "requested-model", "test", "chat") + result, _, _, err := manager.RunSync(ctx, "parent", 0, "test task", "test", "requested-model", "test", "chat") if err != nil { t.Fatalf("RunSync() error = %v", err) } @@ -50,3 +177,147 @@ func TestRunSyncHonorsPerTaskModelOverride(t *testing.T) { t.Fatalf("provider model = %q, want per-task override %q", provider.model, "requested-model") } } + +func TestRunSyncRetainedCompletedTaskDoesNotConsumeActiveQuota(t *testing.T) { + provider := &recordingSubagentProvider{} + manager := NewSubagentManager(provider, nil, "manager-default", nil, NewRegistry, SubagentConfig{ + MaxConcurrent: 1, + MaxSpawnDepth: 3, + MaxChildrenPerAgent: 1, + }) + + ctx := subagentTestContext("parent") + + for i := range 2 { + if _, _, _, err := manager.RunSync( + ctx, "parent", 0, "test task", "test", "", "test", "chat", + ); err != nil { + t.Fatalf("RunSync() attempt %d error = %v", i+1, err) + } + } + + tasks := manager.ListTasks(subagentScopeFromContext(ctx), "") + if len(tasks) != 2 { + t.Fatalf("retained tasks = %d, want 2", len(tasks)) + } + for _, task := range tasks { + if task.Status != TaskStatusCompleted { + t.Fatalf("retained task status = %q, want completed", task.Status) + } + } +} + +func TestSubagentTaskScopeRejectsOtherRootTree(t *testing.T) { + tenantID := uuid.New() + agentA := uuid.New() + agentB := uuid.New() + manager := NewSubagentManager(nil, nil, "", nil, nil, SubagentConfig{}) + manager.tasks["running-a"] = &SubagentTask{ + ID: "running-a", Status: TaskStatusRunning, RootAgentID: agentA, + RootAgentKey: "agent-a", OriginTenantID: tenantID, + } + if _, ok := manager.GetTask(TaskScope{ + TenantID: tenantID, RootAgentID: agentB, RootAgentKey: "agent-b", + }, "running-a"); ok { + t.Fatal("other root tree read task by ID") + } + if manager.CancelTask(TaskScope{ + TenantID: tenantID, RootAgentID: agentB, RootAgentKey: "agent-b", + }, "running-a") { + t.Fatal("other root tree cancelled task") + } +} + +type mediaSubagentProvider struct { + calls int +} + +func (p *mediaSubagentProvider) Name() string { return "media-test" } +func (p *mediaSubagentProvider) DefaultModel() string { return "media-test-model" } +func (p *mediaSubagentProvider) Chat(_ context.Context, _ providers.ChatRequest) (*providers.ChatResponse, error) { + p.calls++ + if p.calls == 1 { + return &providers.ChatResponse{ + ToolCalls: []providers.ToolCall{{ + ID: "create-1", + Name: "create_test_media", + Arguments: map[string]any{}, + }}, + }, nil + } + return &providers.ChatResponse{Content: "media created", FinishReason: "stop"}, nil +} +func (p *mediaSubagentProvider) ChatStream(ctx context.Context, req providers.ChatRequest, _ func(providers.StreamChunk)) (*providers.ChatResponse, error) { + return p.Chat(ctx, req) +} + +type subagentMediaTool struct{} + +func (subagentMediaTool) Name() string { return "create_test_media" } +func (subagentMediaTool) Description() string { return "Create test media" } +func (subagentMediaTool) Parameters() map[string]any { return map[string]any{"type": "object"} } +func (subagentMediaTool) Execute(context.Context, map[string]any) *Result { + return &Result{ + ForLLM: "created", + Media: []bus.MediaFile{{ + Path: "/tmp/subagent-generated.png", + MimeType: "image/png", + Filename: "subagent-generated.png", + }}, + } +} + +func TestRunSyncReturnsGeneratedMedia(t *testing.T) { + provider := &mediaSubagentProvider{} + manager := newMediaSubagentManager(provider) + + ctx := subagentTestContext("parent") + result, media, _, err := manager.RunSync(ctx, "parent", 0, "create media", "media", "", "delegate", "chat") + if err != nil { + t.Fatalf("RunSync() error = %v", err) + } + if result != "media created" { + t.Fatalf("RunSync() result = %q, want %q", result, "media created") + } + if len(media) != 1 || media[0].Path != "/tmp/subagent-generated.png" { + t.Fatalf("RunSync() media = %#v, want generated media", media) + } +} + +func TestSpawnToolSyncForwardsGeneratedMedia(t *testing.T) { + provider := &mediaSubagentProvider{} + tool := NewSpawnTool(newMediaSubagentManager(provider), "parent", 0) + ctx := subagentTestContext("parent") + + result := tool.Execute(ctx, map[string]any{ + "task": "create media", + "mode": "sync", + }) + + if result.IsError { + t.Fatalf("SpawnTool.Execute() error = %s", result.ForLLM) + } + if len(result.Media) != 1 || result.Media[0].Path != "/tmp/subagent-generated.png" { + t.Fatalf("SpawnTool.Execute() media = %#v, want generated media", result.Media) + } +} + +func subagentTestContext(rootKey string) context.Context { + ctx := store.WithTenantID(context.Background(), uuid.New()) + ctx = store.WithAgentID(ctx, uuid.New()) + ctx = store.WithAgentContextWindow(ctx, 200_000) + ctx = store.WithAgentMaxTokens(ctx, 32_000) + return WithToolAgentKey(ctx, rootKey) +} + +func newMediaSubagentManager(provider providers.Provider) *SubagentManager { + return NewSubagentManager(provider, nil, "media-test-model", nil, func() *Registry { + registry := NewRegistry() + registry.Register(subagentMediaTool{}) + return registry + }, SubagentConfig{ + MaxConcurrent: 4, + MaxSpawnDepth: 3, + MaxChildrenPerAgent: 8, + }) +} diff --git a/internal/tools/subagent_spawn_tool.go b/internal/tools/subagent_spawn_tool.go index 48f57332..31be774e 100644 --- a/internal/tools/subagent_spawn_tool.go +++ b/internal/tools/subagent_spawn_tool.go @@ -2,6 +2,7 @@ package tools import ( "context" + "encoding/json" "fmt" "github.com/google/uuid" @@ -39,7 +40,7 @@ func (t *SpawnTool) Parameters() map[string]any { "properties": map[string]any{ "action": map[string]any{ "type": "string", - "description": "'spawn' (default), 'list', 'cancel', 'steer', or 'wait'", + "description": "'spawn' (default), 'get', 'list', 'cancel', 'steer', or 'wait'", }, "task": map[string]any{ "type": "string", @@ -61,6 +62,10 @@ func (t *SpawnTool) Parameters() map[string]any { "type": "string", "description": "Task ID for cancel/steer. For cancel: use 'all' to cancel all or 'last' for most recent", }, + "completion_id": map[string]any{ + "type": "string", + "description": "Durable completion UUID returned by async spawn (required for action=get)", + }, "message": map[string]any{ "type": "string", "description": "New instructions (required for action=steer)", @@ -70,7 +75,6 @@ func (t *SpawnTool) Parameters() map[string]any { "description": "Timeout in seconds for action=wait (default 300)", }, }, - "required": []string{"task"}, } } @@ -81,6 +85,8 @@ func (t *SpawnTool) Execute(ctx context.Context, args map[string]any) *Result { } switch action { + case "get": + return t.executeGet(ctx, args) case "list": return t.executeList(ctx) case "cancel": @@ -89,11 +95,34 @@ func (t *SpawnTool) Execute(ctx context.Context, args map[string]any) *Result { return t.executeSteer(ctx, args) case "wait": return t.executeWait(ctx, args) - default: + case "spawn": return t.executeSpawn(ctx, args) + default: + return ErrorResult(fmt.Sprintf("unknown spawn action %q", action)) } } +func (t *SpawnTool) executeGet(ctx context.Context, args map[string]any) *Result { + rawID, _ := args["completion_id"].(string) + completionID, err := uuid.Parse(rawID) + if err != nil { + return ErrorResult("completion_id must be a valid UUID") + } + scope := subagentScopeFromContext(ctx) + task, err := t.subagentMgr.GetPersistedTask(ctx, scope, completionID) + if err != nil { + return ErrorResult(err.Error()) + } + if task == nil { + return ErrorResult("subagent completion not found") + } + payload, err := json.Marshal(persistedCompletionPayload(task)) + if err != nil { + return ErrorResult("failed to encode subagent completion") + } + return NewResult(string(payload)) +} + func (t *SpawnTool) executeSpawn(ctx context.Context, args map[string]any) *Result { // Reject legacy "agent" parameter — delegation was removed. // Guide the LLM to use team_tasks for team coordination. @@ -116,6 +145,12 @@ func (t *SpawnTool) executeSpawn(ctx context.Context, args map[string]any) *Resu } mode, _ := args["mode"].(string) + if mode == "" { + mode = "async" + } + if err := validateDelegationChildRunMode(ctx, "spawn", mode); err != nil { + return ErrorResult(err.Error()) + } if mode == "sync" { return t.executeSubagentSync(ctx, args, task) } @@ -137,19 +172,56 @@ func (t *SpawnTool) executeSubagentAsync(ctx context.Context, args map[string]an parentID = t.parentID } - msg, err := t.subagentMgr.Spawn(ctx, parentID, t.depth, task, label, modelOverride, + receipt, err := t.subagentMgr.SpawnWithReceipt(ctx, parentID, t.depth, task, label, modelOverride, channel, chatID, peerKind, callback) if err != nil { return ErrorResult(err.Error()) } - forLLM := fmt.Sprintf(`{"status":"accepted","label":%q} + accepted := map[string]any{ + "status": "accepted", + "label": label, + "task_id": receipt.TaskID, + } + if receipt.CompletionID != uuid.Nil { + accepted["completion_id"] = receipt.CompletionID.String() + } + acceptedJSON, _ := json.Marshal(accepted) + forLLM := fmt.Sprintf(`%s %s -After all spawn tool calls in this turn are complete, briefly tell the user what tasks you've started. Subagents will announce results when done — do NOT wait or poll.`, label, msg) +After all spawn tool calls in this turn are complete, briefly tell the user what tasks you've started. Subagents will announce results when done. If an announcement is missed, retrieve the durable result with spawn(action="get", completion_id="..."). Do NOT wait or poll while the task is running.`, acceptedJSON, receipt.Message) return AsyncResult(forLLM) } +func persistedCompletionPayload(task *store.SubagentTaskData) map[string]any { + payload := map[string]any{ + "completion_id": task.ID.String(), + "status": task.Status, + "subject": task.Subject, + "created_at": task.CreatedAt, + "updated_at": task.UpdatedAt, + } + if task.Result != nil { + payload["result"] = *task.Result + } + if task.CompletedAt != nil { + payload["completed_at"] = *task.CompletedAt + } + if task.Metadata != nil { + if runtimeID, ok := task.Metadata[asyncCompletionRuntimeIDKey].(string); ok && runtimeID != "" { + payload["task_id"] = runtimeID + } + if delivery, ok := task.Metadata[asyncCompletionDeliveryKey].(string); ok && delivery != "" { + payload[asyncCompletionDeliveryKey] = delivery + } + if media := persistedCompletionMediaPayload(task.Metadata[asyncCompletionMediaKey]); len(media) > 0 { + payload["media"] = media + } + } + return payload +} + // executeSubagentSync runs a sync self-clone. func (t *SpawnTool) executeSubagentSync(ctx context.Context, args map[string]any, task string) *Result { label, _ := args["label"].(string) @@ -166,7 +238,7 @@ func (t *SpawnTool) executeSubagentSync(ctx context.Context, args map[string]any parentID = t.parentID } - result, iterations, err := t.subagentMgr.RunSync(ctx, parentID, t.depth, task, label, modelOverride, + result, media, iterations, err := t.subagentMgr.RunSync(ctx, parentID, t.depth, task, label, modelOverride, channel, chatID) if err != nil { return ErrorResult(fmt.Sprintf("Subagent '%s' failed: %v", label, err)) @@ -182,7 +254,7 @@ func (t *SpawnTool) executeSubagentSync(ctx context.Context, args map[string]any forLLM := fmt.Sprintf("Subagent '%s' completed in %d iterations.\n\nFull result:\n%s", label, iterations, result) - return &Result{ForLLM: forLLM, ForUser: forUser} + return &Result{ForLLM: forLLM, ForUser: forUser, Media: media} } // SetContext is a no-op; channel/chatID are now read from ctx (thread-safe). @@ -193,3 +265,11 @@ func (t *SpawnTool) SetPeerKind(peerKind string) {} // SetCallback is a no-op; callback is now read from ctx (thread-safe). func (t *SpawnTool) SetCallback(cb AsyncCallback) {} + +func (t *SpawnTool) scopeFromContext(ctx context.Context) TaskScope { + scope := subagentScopeFromContext(ctx) + if scope.RootAgentKey == "" { + scope.RootAgentKey = t.parentID + } + return scope +} diff --git a/internal/tools/subagent_spawn_tool_actions.go b/internal/tools/subagent_spawn_tool_actions.go index b618a658..f190e4b3 100644 --- a/internal/tools/subagent_spawn_tool_actions.go +++ b/internal/tools/subagent_spawn_tool_actions.go @@ -7,26 +7,29 @@ import ( "time" ) -// executeList shows active subagent tasks. +// executeList shows running and retained subagent tasks. func (t *SpawnTool) executeList(ctx context.Context) *Result { - parentID := ToolAgentKeyFromCtx(ctx) - if parentID == "" { - parentID = t.parentID - } - tasks := t.subagentMgr.ListTasks(parentID) + scope := t.scopeFromContext(ctx) + tasks := t.subagentMgr.ListTasks(scope, "") if len(tasks) == 0 { - return &Result{ForLLM: "No active tasks found."} + return &Result{ForLLM: "No subagent tasks found."} } var lines []string - running, completed, cancelled := 0, 0, 0 + queued, running, completed, failed, cancelled := 0, 0, 0, 0, 0 for _, task := range tasks { switch task.Status { - case "running": + case TaskStatusQueued: + queued++ + case TaskStatusRunning: running++ - case "completed": + case TaskStatusWaiting: + running++ + case TaskStatusCompleted: completed++ - case "cancelled": + case TaskStatusFailed: + failed++ + case TaskStatusCancelled: cancelled++ } line := fmt.Sprintf("- [%s] %s (id=%s, status=%s)", task.Label, truncate(task.Task, 60), task.ID, task.Status) @@ -37,8 +40,10 @@ func (t *SpawnTool) executeList(ctx context.Context) *Result { lines = append(lines, line) } - return &Result{ForLLM: fmt.Sprintf("Subagent tasks: %d running, %d completed, %d cancelled\n%s", - running, completed, cancelled, strings.Join(lines, "\n"))} + return &Result{ForLLM: fmt.Sprintf( + "Subagent tasks: %d queued, %d active, %d completed, %d failed, %d cancelled (%d retained total)\n%s", + queued, running, completed, failed, cancelled, len(tasks), strings.Join(lines, "\n"), + )} } // executeCancel cancels a subagent task by ID. @@ -48,7 +53,7 @@ func (t *SpawnTool) executeCancel(ctx context.Context, args map[string]any) *Res return ErrorResult("id is required for action=cancel") } - if t.subagentMgr.CancelTask(id) { + if t.subagentMgr.CancelTask(t.scopeFromContext(ctx), id) { return &Result{ForLLM: fmt.Sprintf("Task '%s' cancelled.", id)} } @@ -66,7 +71,7 @@ func (t *SpawnTool) executeSteer(ctx context.Context, args map[string]any) *Resu return ErrorResult("message is required for action=steer") } - msg, err := t.subagentMgr.Steer(ctx, id, message, nil) + msg, err := t.subagentMgr.Steer(ctx, t.scopeFromContext(ctx), id, message, nil) if err != nil { return ErrorResult(err.Error()) } @@ -75,17 +80,12 @@ func (t *SpawnTool) executeSteer(ctx context.Context, args map[string]any) *Resu // executeWait blocks until all children of the calling agent complete or timeout. func (t *SpawnTool) executeWait(ctx context.Context, args map[string]any) *Result { - parentID := ToolAgentKeyFromCtx(ctx) - if parentID == "" { - parentID = t.parentID - } - timeout := 300 if v, ok := args["timeout"].(float64); ok && v > 0 { timeout = int(v) } - tasks, err := t.subagentMgr.WaitForChildren(ctx, parentID, timeout) + tasks, err := t.subagentMgr.WaitForChildren(ctx, t.scopeFromContext(ctx), "", timeout) return t.formatWaitResult(tasks, err) } diff --git a/internal/tools/subagent_task_lifecycle_test.go b/internal/tools/subagent_task_lifecycle_test.go new file mode 100644 index 00000000..d44e7039 --- /dev/null +++ b/internal/tools/subagent_task_lifecycle_test.go @@ -0,0 +1,552 @@ +package tools + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/google/uuid" + + orchestration "github.com/nextlevelbuilder/goclaw/internal/childrun" + "github.com/nextlevelbuilder/goclaw/internal/providers" + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +type taskLifecycleUpdate struct { + rootAgentID uuid.UUID + id uuid.UUID + status string + result string +} + +type recordingSubagentTaskStore struct { + mu sync.Mutex + creates []store.SubagentTaskData + rows map[uuid.UUID]store.SubagentTaskData + updates chan taskLifecycleUpdate + metadata chan map[string]any + createStarted chan struct{} + createRelease chan struct{} + createErr error + updateErr error + updateHook func(context.Context, string) error + updateStarted chan struct{} + updateRelease chan struct{} + updateOnce sync.Once +} + +func newRecordingSubagentTaskStore() *recordingSubagentTaskStore { + return &recordingSubagentTaskStore{ + rows: make(map[uuid.UUID]store.SubagentTaskData), + updates: make(chan taskLifecycleUpdate, 16), + metadata: make(chan map[string]any, 16), + } +} + +func TestSubagentTaskStatusesFitPersistentStore(t *testing.T) { + statuses := []string{ + TaskStatusQueued, + TaskStatusRunning, + TaskStatusWaiting, + TaskStatusCompleted, + TaskStatusFailed, + TaskStatusCancelled, + } + for _, status := range statuses { + if len(status) > store.SubagentTaskStatusMaxLength { + t.Fatalf( + "subagent status %q is %d characters, exceeds persistent limit %d", + status, + len(status), + store.SubagentTaskStatusMaxLength, + ) + } + } + if TaskStatusWaiting != string(orchestration.ChildRunWaitingChild) { + t.Fatalf( + "task waiting status %q differs from child-run state %q", + TaskStatusWaiting, + orchestration.ChildRunWaitingChild, + ) + } +} + +func (s *recordingSubagentTaskStore) Create(_ context.Context, task *store.SubagentTaskData) error { + if s.createErr != nil { + return s.createErr + } + s.mu.Lock() + s.creates = append(s.creates, *task) + s.rows[task.ID] = *task + s.mu.Unlock() + if s.createStarted != nil { + close(s.createStarted) + <-s.createRelease + } + return nil +} + +func (s *recordingSubagentTaskStore) Get(_ context.Context, rootAgentID, id uuid.UUID) (*store.SubagentTaskData, error) { + s.mu.Lock() + defer s.mu.Unlock() + task, ok := s.rows[id] + if !ok || task.RootAgentID != rootAgentID { + return nil, nil + } + copy := task + return ©, nil +} + +func (s *recordingSubagentTaskStore) UpdateStatus( + ctx context.Context, + rootAgentID uuid.UUID, + id uuid.UUID, + status string, + result *string, + _ int, + _, _ int64, +) error { + value := "" + if result != nil { + value = *result + } + s.updates <- taskLifecycleUpdate{ + rootAgentID: rootAgentID, + id: id, + status: status, + result: value, + } + if s.updateHook != nil { + if err := s.updateHook(ctx, status); err != nil { + return err + } + } + if s.updateErr != nil { + return s.updateErr + } + s.mu.Lock() + if task, ok := s.rows[id]; ok && task.RootAgentID == rootAgentID { + task.Status = status + task.Result = result + s.rows[id] = task + } + s.mu.Unlock() + if s.updateStarted != nil { + s.updateOnce.Do(func() { close(s.updateStarted) }) + <-s.updateRelease + } + return nil +} + +func (s *recordingSubagentTaskStore) ListByParent(context.Context, uuid.UUID, string) ([]store.SubagentTaskData, error) { + return nil, nil +} + +func (s *recordingSubagentTaskStore) ListBySession(context.Context, uuid.UUID, string) ([]store.SubagentTaskData, error) { + return nil, nil +} + +func (s *recordingSubagentTaskStore) Archive(context.Context, uuid.UUID, time.Duration, int) (int64, error) { + return 0, nil +} + +func (s *recordingSubagentTaskStore) UpdateMetadata(_ context.Context, rootAgentID, id uuid.UUID, metadata map[string]any) error { + s.mu.Lock() + if task, ok := s.rows[id]; ok && task.RootAgentID == rootAgentID { + if task.Metadata == nil { + task.Metadata = make(map[string]any) + } + for key, value := range metadata { + task.Metadata[key] = value + } + s.rows[id] = task + } + s.mu.Unlock() + s.metadata <- metadata + return nil +} + +type blockingSubagentProvider struct { + started chan struct{} + release chan struct{} +} + +func (*blockingSubagentProvider) Name() string { return "blocking" } +func (*blockingSubagentProvider) DefaultModel() string { return "blocking" } +func (p *blockingSubagentProvider) Chat( + context.Context, + providers.ChatRequest, +) (*providers.ChatResponse, error) { + p.started <- struct{}{} + <-p.release + return &providers.ChatResponse{Content: "done", FinishReason: "stop"}, nil +} +func (p *blockingSubagentProvider) ChatStream( + ctx context.Context, + req providers.ChatRequest, + _ func(providers.StreamChunk), +) (*providers.ChatResponse, error) { + return p.Chat(ctx, req) +} + +func TestRootSubagentsHonorMaxChildrenFanout(t *testing.T) { + provider := &blockingSubagentProvider{ + started: make(chan struct{}, 2), + release: make(chan struct{}), + } + admission := orchestration.NewChildRunAdmission(2, 4) + manager := NewSubagentManagerWithAdmission( + provider, + nil, + "model", + nil, + NewRegistry, + SubagentConfig{MaxConcurrent: 2, MaxSpawnDepth: 3, MaxChildrenPerAgent: 1}, + admission, + ) + ctx := subagentTestContext("parent") + + if _, err := manager.Spawn(ctx, "parent", 0, "first", "first", "", "test", "chat", "", nil); err != nil { + t.Fatalf("spawn first: %v", err) + } + select { + case <-provider.started: + case <-time.After(time.Second): + t.Fatal("first root child did not start") + } + if _, err := manager.Spawn(ctx, "parent", 0, "second", "second", "", "test", "chat", "", nil); err != nil { + t.Fatalf("spawn second: %v", err) + } + select { + case <-provider.started: + t.Fatal("second root child bypassed maxChildren fanout") + case <-time.After(100 * time.Millisecond): + } + + close(provider.release) + select { + case <-provider.started: + case <-time.After(time.Second): + t.Fatal("queued root child did not start after capacity released") + } + if err := admission.Close(context.Background()); err != nil { + t.Fatal(err) + } + manager.Close() +} + +func TestRunSyncAdmissionRejectionCreatesNoGhostTask(t *testing.T) { + admission := orchestration.NewChildRunAdmission(1, 1) + if err := admission.Close(context.Background()); err != nil { + t.Fatal(err) + } + provider := &recordingSubagentProvider{} + manager := NewSubagentManagerWithAdmission( + provider, + nil, + "model", + nil, + NewRegistry, + SubagentConfig{MaxConcurrent: 2, MaxSpawnDepth: 3, MaxChildrenPerAgent: 2}, + admission, + ) + taskStore := newRecordingSubagentTaskStore() + manager.SetTaskStore(taskStore) + + ctx := subagentTestContext("parent") + if _, _, _, err := manager.RunSync(ctx, "parent", 0, "task", "label", "", "test", "chat"); err == nil { + t.Fatal("closed admission unexpectedly accepted sync task") + } + select { + case update := <-taskStore.updates: + t.Fatalf("rejected task persisted an update: %#v", update) + default: + } + taskStore.mu.Lock() + if len(taskStore.creates) != 0 { + t.Fatalf("rejected task persisted creates: %#v", taskStore.creates) + } + taskStore.mu.Unlock() + if got := manager.ListTasks(subagentScopeFromContext(ctx), ""); len(got) != 0 { + t.Fatalf("rejected task remained in memory: %#v", got) + } + if provider.calls != 0 { + t.Fatalf("provider calls = %d, want 0", provider.calls) + } +} + +func TestCancelQueuedTaskPersistsCancelledStatus(t *testing.T) { + admission := orchestration.NewChildRunAdmission(1, 4) + blockerStarted := make(chan struct{}) + releaseBlocker := make(chan struct{}) + blocker, err := admission.Enqueue(context.Background(), orchestration.ChildRunConstraints{ + TenantID: store.MasterTenantID, + TaskID: "blocker", + }, func(context.Context, *orchestration.ChildRunLease) { + close(blockerStarted) + <-releaseBlocker + }) + if err != nil { + t.Fatal(err) + } + if err := blocker.Activate(); err != nil { + t.Fatal(err) + } + select { + case <-blockerStarted: + case <-time.After(time.Second): + t.Fatal("blocker did not start") + } + + manager := NewSubagentManagerWithAdmission( + &recordingSubagentProvider{}, + nil, + "model", + nil, + NewRegistry, + SubagentConfig{MaxConcurrent: 2, MaxSpawnDepth: 3, MaxChildrenPerAgent: 2}, + admission, + ) + taskStore := newRecordingSubagentTaskStore() + manager.SetTaskStore(taskStore) + ctx := subagentTestContext("parent") + if _, err := manager.Spawn(ctx, "parent", 0, "queued task", "queued", "", "test", "chat", "", nil); err != nil { + t.Fatalf("spawn queued task: %v", err) + } + scope := subagentScopeFromContext(ctx) + tasks := manager.ListTasks(scope, "") + if len(tasks) != 1 || tasks[0].Status != TaskStatusQueued { + t.Fatalf("queued tasks = %#v", tasks) + } + taskStore.mu.Lock() + if len(taskStore.creates) != 1 || taskStore.creates[0].RootAgentID != scope.RootAgentID { + t.Fatalf("persisted create root ownership = %#v, want %s", taskStore.creates, scope.RootAgentID) + } + taskStore.mu.Unlock() + if !manager.CancelTask(scope, tasks[0].ID) { + t.Fatal("queued task was not cancelled") + } + select { + case update := <-taskStore.updates: + if update.rootAgentID != scope.RootAgentID || + update.status != TaskStatusCancelled || update.result != "cancelled by user" { + t.Fatalf("cancel update = %#v", update) + } + case <-time.After(time.Second): + t.Fatal("queued cancellation was not persisted") + } + + close(releaseBlocker) + <-blocker.Done() + if err := admission.Close(context.Background()); err != nil { + t.Fatal(err) + } + manager.Close() +} + +func TestManagerCloseWaitsForQueuedCancellationPersistence(t *testing.T) { + admission := orchestration.NewChildRunAdmission(1, 4) + blockerStarted := make(chan struct{}) + releaseBlocker := make(chan struct{}) + blocker, err := admission.Enqueue(context.Background(), orchestration.ChildRunConstraints{ + TenantID: store.MasterTenantID, + TaskID: "blocker", + }, func(context.Context, *orchestration.ChildRunLease) { + close(blockerStarted) + <-releaseBlocker + }) + if err != nil { + t.Fatal(err) + } + if err := blocker.Activate(); err != nil { + t.Fatal(err) + } + select { + case <-blockerStarted: + case <-time.After(time.Second): + t.Fatal("blocker did not start") + } + + manager := NewSubagentManagerWithAdmission( + &recordingSubagentProvider{}, + nil, + "model", + nil, + NewRegistry, + SubagentConfig{MaxConcurrent: 2, MaxSpawnDepth: 3, MaxChildrenPerAgent: 2}, + admission, + ) + taskStore := newRecordingSubagentTaskStore() + taskStore.updateStarted = make(chan struct{}) + taskStore.updateRelease = make(chan struct{}) + manager.SetTaskStore(taskStore) + ctx := subagentTestContext("parent") + if _, err := manager.Spawn(ctx, "parent", 0, "queued task", "queued", "", "test", "chat", "", nil); err != nil { + t.Fatalf("spawn queued task: %v", err) + } + scope := subagentScopeFromContext(ctx) + tasks := manager.ListTasks(scope, "") + if len(tasks) != 1 { + t.Fatalf("queued tasks = %#v", tasks) + } + if !manager.CancelTask(scope, tasks[0].ID) { + t.Fatal("queued task was not cancelled") + } + select { + case <-taskStore.updateStarted: + case <-time.After(time.Second): + t.Fatal("terminal persistence did not start") + } + + close(releaseBlocker) + <-blocker.Done() + if err := admission.Close(context.Background()); err != nil { + t.Fatal(err) + } + closeDone := make(chan struct{}) + go func() { + manager.Close() + close(closeDone) + }() + select { + case <-closeDone: + t.Fatal("manager closed before terminal persistence finished") + case <-time.After(100 * time.Millisecond): + } + close(taskStore.updateRelease) + select { + case <-closeDone: + case <-time.After(time.Second): + t.Fatal("manager did not close after terminal persistence finished") + } +} + +func TestManagerCloseContextTimesOutOnBlockedTerminalPersistence(t *testing.T) { + manager := NewSubagentManagerWithAdmission( + &recordingSubagentProvider{}, + nil, + "model", + nil, + NewRegistry, + SubagentConfig{MaxConcurrent: 1, MaxSpawnDepth: 1, MaxChildrenPerAgent: 1}, + orchestration.NewChildRunAdmission(1, 1), + ) + finish, ok := manager.beginLifecycleOperation() + if !ok { + t.Fatal("manager rejected lifecycle operation before close") + } + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + err := manager.CloseContext(ctx) + if !errors.Is(err, ErrSubagentLifecycleDrainTimeout) { + t.Fatalf("CloseContext error = %v, want typed drain timeout", err) + } + + finish() + if err := manager.CloseContext(context.Background()); err != nil { + t.Fatalf("CloseContext retry: %v", err) + } +} + +func TestManagerCloseContextDropsPendingAnnounceBeforeLifecycleDrain(t *testing.T) { + manager := NewSubagentManagerWithAdmission( + &recordingSubagentProvider{}, + nil, + "model", + nil, + NewRegistry, + SubagentConfig{MaxConcurrent: 1, MaxSpawnDepth: 1, MaxChildrenPerAgent: 1}, + orchestration.NewChildRunAdmission(1, 1), + ) + drained := make(chan struct{}, 1) + manager.SetAnnounceQueue(NewAnnounceQueue( + 20, + 20, + func(string, []AnnounceQueueItem, AnnounceMetadata) { + drained <- struct{}{} + }, + )) + finish, ok := manager.beginLifecycleOperation() + if !ok { + t.Fatal("manager rejected lifecycle operation before close") + } + manager.announceQueue.Enqueue( + "session", + AnnounceQueueItem{SubagentID: "task"}, + AnnounceMetadata{}, + ) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + err := manager.CloseContext(ctx) + cancel() + if !errors.Is(err, ErrSubagentLifecycleDrainTimeout) { + t.Fatalf("CloseContext error = %v, want typed drain timeout", err) + } + select { + case <-drained: + t.Fatal("pending announce drained while lifecycle shutdown was blocked") + case <-time.After(50 * time.Millisecond): + } + + finish() + if err := manager.CloseContext(context.Background()); err != nil { + t.Fatalf("CloseContext retry: %v", err) + } +} + +func TestSpawnActivationFailureTerminalizesRowAndRemovesMemoryTask(t *testing.T) { + admission := orchestration.NewChildRunAdmission(1, 4) + manager := NewSubagentManagerWithAdmission( + &recordingSubagentProvider{}, + nil, + "model", + nil, + NewRegistry, + SubagentConfig{MaxConcurrent: 2, MaxSpawnDepth: 3, MaxChildrenPerAgent: 2}, + admission, + ) + taskStore := newRecordingSubagentTaskStore() + taskStore.createStarted = make(chan struct{}) + taskStore.createRelease = make(chan struct{}) + manager.SetTaskStore(taskStore) + + ctx := subagentTestContext("parent") + spawnDone := make(chan error, 1) + go func() { + _, err := manager.Spawn(ctx, "parent", 0, "task", "label", "", "test", "chat", "", nil) + spawnDone <- err + }() + select { + case <-taskStore.createStarted: + case <-time.After(time.Second): + t.Fatal("queued row was not persisted") + } + if err := admission.Close(context.Background()); err != nil { + t.Fatalf("close admission: %v", err) + } + close(taskStore.createRelease) + select { + case err := <-spawnDone: + if err == nil { + t.Fatal("activation unexpectedly succeeded after admission close") + } + case <-time.After(time.Second): + t.Fatal("spawn did not return after activation failure") + } + + select { + case update := <-taskStore.updates: + if update.status != TaskStatusFailed || update.result == "" { + t.Fatalf("activation rollback update = %#v", update) + } + case <-time.After(time.Second): + t.Fatal("activation failure did not terminalize persisted row") + } + if got := manager.ListTasks(subagentScopeFromContext(ctx), ""); len(got) != 0 { + t.Fatalf("activation failure left in-memory task: %#v", got) + } + manager.Close() +} diff --git a/internal/tools/subagent_tracing.go b/internal/tools/subagent_tracing.go index 1f5f3254..0bea5a84 100644 --- a/internal/tools/subagent_tracing.go +++ b/internal/tools/subagent_tracing.go @@ -52,7 +52,7 @@ func (sm *SubagentManager) emitLLMSpanStart(ctx context.Context, start time.Time span.InputPreview = truncate(string(b), 100000) } } - collector.EmitSpan(span) + collector.EmitSpan(tracing.RedactSpan(ctx, span)) return spanID } @@ -97,7 +97,7 @@ func (sm *SubagentManager) emitLLMSpanEnd(ctx context.Context, spanID uuid.UUID, } updates["output_preview"] = truncate(resp.Content, previewLimit) } - collector.EmitSpanUpdate(spanID, traceID, updates) + collector.EmitSpanUpdate(spanID, traceID, tracing.RedactSpanUpdates(ctx, updates)) } // --------------------------------------------------------------------------- @@ -139,7 +139,7 @@ func (sm *SubagentManager) emitToolSpanStart(ctx context.Context, start time.Tim if span.TenantID == uuid.Nil { span.TenantID = store.MasterTenantID } - collector.EmitSpan(span) + collector.EmitSpan(tracing.RedactSpan(ctx, span)) return spanID } @@ -169,7 +169,7 @@ func (sm *SubagentManager) emitToolSpanEnd(ctx context.Context, spanID uuid.UUID updates["status"] = store.SpanStatusError updates["error"] = truncate(output, 200) } - collector.EmitSpanUpdate(spanID, traceID, updates) + collector.EmitSpanUpdate(spanID, traceID, tracing.RedactSpanUpdates(ctx, updates)) } // --------------------------------------------------------------------------- @@ -210,7 +210,7 @@ func (sm *SubagentManager) emitSubagentSpanStart(ctx context.Context, spanID uui if span.TenantID == uuid.Nil { span.TenantID = store.MasterTenantID } - collector.EmitSpan(span) + collector.EmitSpan(tracing.RedactSpan(ctx, span)) } // emitSubagentSpanEnd finalizes the running subagent root span. @@ -239,5 +239,5 @@ func (sm *SubagentManager) emitSubagentSpanEnd(ctx context.Context, spanID uuid. updates["status"] = store.SpanStatusError updates["error"] = truncate(task.Result, 200) } - collector.EmitSpanUpdate(spanID, traceID, updates) + collector.EmitSpanUpdate(spanID, traceID, tracing.RedactSpanUpdates(ctx, updates)) } diff --git a/internal/tools/team_metadata_keys.go b/internal/tools/team_metadata_keys.go index b2177baf..3d655e1e 100644 --- a/internal/tools/team_metadata_keys.go +++ b/internal/tools/team_metadata_keys.go @@ -4,46 +4,49 @@ package tools // These keys appear in bus.InboundMessage.Metadata for teammate and // subagent messages routed through the gateway consumer. const ( - MetaOriginChannel = "origin_channel" - MetaOriginPeerKind = "origin_peer_kind" - MetaOriginChatID = "origin_chat_id" - MetaOriginUserID = "origin_user_id" + MetaOriginChannel = "origin_channel" + MetaOriginPeerKind = "origin_peer_kind" + MetaOriginChatID = "origin_chat_id" + MetaOriginUserID = "origin_user_id" // MetaOriginSenderID carries the real acting sender through announce re-ingress // so permission checks (e.g. CheckFileWriterPermission) attribute to the // original user rather than a synthetic "subagent:" / "notification:system" string. - MetaOriginSenderID = "origin_sender_id" + MetaOriginSenderID = "origin_sender_id" // MetaOriginRole carries the caller's RBAC role through dispatch + re-ingress // so permission checks can bypass per-user grants for authenticated admins // (e.g. dashboard user dispatches a task that writes files in a group chat). - MetaOriginRole = "origin_role" - MetaOriginLocalKey = "origin_local_key" - MetaOriginSessionKey = "origin_session_key" - MetaOriginTraceID = "origin_trace_id" - MetaOriginRootSpanID = "origin_root_span_id" - MetaFromAgent = "from_agent" - MetaToAgent = "to_agent" - MetaToAgentDisplay = "to_agent_display" - MetaTeamTaskID = "team_task_id" - MetaTeamID = "team_id" - MetaTeamWorkspace = "team_workspace" - MetaLeaderAgentID = "leader_agent_id" - MetaParentAgent = "parent_agent" - MetaSubagentLabel = "subagent_label" - MetaSubagentStatus = "subagent_status" - MetaSubagentResult = "subagent_result" - MetaSubagentRuntime = "subagent_runtime_ms" - MetaSubagentIterations = "subagent_iterations" - MetaSubagentInputToks = "subagent_input_tokens" - MetaSubagentOutputToks = "subagent_output_tokens" - MetaCommand = "command" - MetaIsForum = "is_forum" - MetaMessageThreadID = "message_thread_id" - MetaDMThreadID = "dm_thread_id" - MetaChatTitle = "chat_title" - MetaUsername = "username" - MetaUserName = "user_name" - MetaTopicSystemPrompt = "topic_system_prompt" - MetaTopicSkills = "topic_skills" + MetaOriginRole = "origin_role" + MetaOriginLocalKey = "origin_local_key" + MetaOriginSessionKey = "origin_session_key" + MetaOriginTraceID = "origin_trace_id" + MetaOriginRootSpanID = "origin_root_span_id" + MetaFromAgent = "from_agent" + MetaToAgent = "to_agent" + MetaToAgentDisplay = "to_agent_display" + MetaTeamTaskID = "team_task_id" + MetaTeamID = "team_id" + MetaTeamWorkspace = "team_workspace" + MetaLeaderAgentID = "leader_agent_id" + MetaParentAgent = "parent_agent" + MetaSubagentRootAgentID = "subagent_root_agent_id" + MetaSubagentParentTask = "subagent_parent_task_id" + MetaSubagentDepth = "subagent_depth" + MetaSubagentLabel = "subagent_label" + MetaSubagentStatus = "subagent_status" + MetaSubagentResult = "subagent_result" + MetaSubagentRuntime = "subagent_runtime_ms" + MetaSubagentIterations = "subagent_iterations" + MetaSubagentInputToks = "subagent_input_tokens" + MetaSubagentOutputToks = "subagent_output_tokens" + MetaCommand = "command" + MetaIsForum = "is_forum" + MetaMessageThreadID = "message_thread_id" + MetaDMThreadID = "dm_thread_id" + MetaChatTitle = "chat_title" + MetaUsername = "username" + MetaUserName = "user_name" + MetaTopicSystemPrompt = "topic_system_prompt" + MetaTopicSkills = "topic_skills" // MetaChannelSelfIdentity carries a channel-provided self-identity hint // (e.g. "You are @viet_super_bot (ViệtBot) on this Telegram channel.") // appended to the agent's system prompt so the LLM does not confuse its own diff --git a/internal/tracing/redaction.go b/internal/tracing/redaction.go new file mode 100644 index 00000000..3335d07d --- /dev/null +++ b/internal/tracing/redaction.go @@ -0,0 +1,114 @@ +package tracing + +import ( + "context" + "encoding/json" + + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +type redactorContextKey struct{} + +// TextRedactor removes runtime-only values before they cross a trace, event, or +// persistence boundary. It must be deterministic and safe to call repeatedly. +type TextRedactor func(string) string + +// WithTextRedactor adds a redactor while preserving any boundary installed by +// an outer run. +func WithTextRedactor(ctx context.Context, redactor TextRedactor) context.Context { + if redactor == nil { + return ctx + } + if outer := textRedactorFromContext(ctx); outer != nil { + inner := redactor + redactor = func(value string) string { + return inner(outer(value)) + } + } + return context.WithValue(ctx, redactorContextKey{}, redactor) +} + +func textRedactorFromContext(ctx context.Context) TextRedactor { + if ctx == nil { + return nil + } + redactor, _ := ctx.Value(redactorContextKey{}).(TextRedactor) + return redactor +} + +func RedactText(ctx context.Context, value string) string { + return RedactTextWith(textRedactorFromContext(ctx), value) +} + +func RedactTextWith(redactor TextRedactor, value string) string { + if redactor == nil || value == "" { + return value + } + return redactor(value) +} + +// RedactValue preserves the common payload shapes used by agent events and +// trace metadata while redacting every contained string. +func RedactValue(ctx context.Context, value any) any { + return RedactValueWith(textRedactorFromContext(ctx), value) +} + +func RedactValueWith(redactor TextRedactor, value any) any { + if redactor == nil || value == nil { + return value + } + switch typed := value.(type) { + case string: + return redactor(typed) + case json.RawMessage: + return json.RawMessage(redactor(string(typed))) + case []byte: + return []byte(redactor(string(typed))) + case map[string]string: + copyValue := make(map[string]string, len(typed)) + for key, item := range typed { + copyValue[key] = redactor(item) + } + return copyValue + case map[string]any: + copyValue := make(map[string]any, len(typed)) + for key, item := range typed { + copyValue[key] = RedactValueWith(redactor, item) + } + return copyValue + case []string: + copyValue := make([]string, len(typed)) + for i, item := range typed { + copyValue[i] = redactor(item) + } + return copyValue + case []any: + copyValue := make([]any, len(typed)) + for i, item := range typed { + copyValue[i] = RedactValueWith(redactor, item) + } + return copyValue + default: + return value + } +} + +func RedactSpan(ctx context.Context, span store.SpanData) store.SpanData { + redactor := textRedactorFromContext(ctx) + if redactor == nil { + return span + } + span.Name = redactor(span.Name) + span.Error = redactor(span.Error) + span.InputPreview = redactor(span.InputPreview) + span.OutputPreview = redactor(span.OutputPreview) + if len(span.Metadata) > 0 { + span.Metadata = json.RawMessage(redactor(string(span.Metadata))) + } + return span +} + +func RedactSpanUpdates(ctx context.Context, updates map[string]any) map[string]any { + redacted, _ := RedactValue(ctx, updates).(map[string]any) + return redacted +} diff --git a/internal/tracing/redaction_test.go b/internal/tracing/redaction_test.go new file mode 100644 index 00000000..c5fee69e --- /dev/null +++ b/internal/tracing/redaction_test.go @@ -0,0 +1,47 @@ +package tracing + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +func TestTextRedactorSanitizesSpanUpdatesAndEventPayloads(t *testing.T) { + const secret = "/host/tenant/collaboration/delegations/id/outputs" + ctx := WithTextRedactor(context.Background(), func(value string) string { + return strings.ReplaceAll(value, secret, "outputs") + }) + + span := RedactSpan(ctx, store.SpanData{ + InputPreview: `{"path":"` + secret + `/input.txt"}`, + OutputPreview: secret + "/result.txt", + Error: "failed at " + secret, + Metadata: json.RawMessage(`{"root":"` + secret + `"}`), + }) + encoded, err := json.Marshal(span) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(encoded), secret) { + t.Fatalf("redacted span leaked host root: %s", encoded) + } + + updates := RedactSpanUpdates(ctx, map[string]any{ + "output_preview": secret + "/result.txt", + "metadata": json.RawMessage(`{"root":"` + secret + `"}`), + }) + payload := RedactValue(ctx, map[string]any{ + "content": secret, + "items": []any{secret + "/child"}, + }) + combined, err := json.Marshal([]any{updates, payload}) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(combined), secret) { + t.Fatalf("redacted values leaked host root: %s", combined) + } +} diff --git a/internal/upgrade/version.go b/internal/upgrade/version.go index f3affaad..bc2504b5 100644 --- a/internal/upgrade/version.go +++ b/internal/upgrade/version.go @@ -2,4 +2,4 @@ package upgrade // RequiredSchemaVersion is the schema migration version this binary requires. // Bump this whenever adding a new SQL migration file. -const RequiredSchemaVersion uint = 95 +const RequiredSchemaVersion uint = 96 diff --git a/internal/workspace/resolver_impl.go b/internal/workspace/resolver_impl.go index 0592daeb..464a4753 100644 --- a/internal/workspace/resolver_impl.go +++ b/internal/workspace/resolver_impl.go @@ -29,10 +29,10 @@ func (r *defaultResolver) Resolve(_ context.Context, params ResolveParams) (*Wor return nil, fmt.Errorf("workspace: base dir is required") } - // Priority: delegation > team > personal/predefined + // Agent Link artifact runs construct their fail-closed workspace directly + // from the admitted exchange. This resolver handles durable team/personal + // workspaces only. switch { - case params.DelegateCtx != nil: - return r.resolveDelegate(params) case params.TeamID != nil && *params.TeamID != "": return r.resolveTeam(params), nil default: @@ -40,30 +40,6 @@ func (r *defaultResolver) Resolve(_ context.Context, params ResolveParams) (*Wor } } -// resolveDelegate handles delegated task workspace. -// ActivePath = delegate's shared path, read-only exports from delegator. -// Validates SharedPath is under BaseDir to prevent directory traversal. -func (r *defaultResolver) resolveDelegate(p ResolveParams) (*WorkspaceContext, error) { - shared := filepath.Clean(p.DelegateCtx.SharedPath) - base := filepath.Clean(p.BaseDir) - if !strings.HasPrefix(shared+string(filepath.Separator), base+string(filepath.Separator)) { - return nil, fmt.Errorf("workspace: delegate shared path escapes base dir") - } - - wc := &WorkspaceContext{ - ActivePath: shared, - Scope: ScopeDelegate, - ReadOnlyPaths: p.DelegateCtx.ExportPaths, - SharedPath: &p.DelegateCtx.SharedPath, - OwnerID: p.UserID, - MemoryScope: "user", - KGScope: "user", - EnforcementLabel: DefaultEnforcementLabel(ScopeDelegate, false), - } - ensureDir(wc.ActivePath) - return wc, nil -} - // resolveTeam handles team workspace (shared or isolated). func (r *defaultResolver) resolveTeam(p ResolveParams) *WorkspaceContext { base := tenantPath(p.BaseDir, p.TenantID, p.TenantSlug) diff --git a/internal/workspace/resolver_impl_test.go b/internal/workspace/resolver_impl_test.go index ca851bfd..ac9fbfbe 100644 --- a/internal/workspace/resolver_impl_test.go +++ b/internal/workspace/resolver_impl_test.go @@ -195,55 +195,6 @@ func TestResolve_TeamIsolated(t *testing.T) { } } -func TestResolve_Delegation(t *testing.T) { - base := t.TempDir() - sharedPath := filepath.Join(base, "shared-task") - exportPath := filepath.Join(base, "exports") - - r := NewResolver() - wc, err := r.Resolve(context.Background(), ResolveParams{ - AgentID: "agent-1", - AgentType: "open", - UserID: "user-1", - BaseDir: base, - DelegateCtx: &DelegateContext{ - LinkID: "link-1", - SharedPath: sharedPath, - ExportPaths: []string{exportPath}, - }, - }) - if err != nil { - t.Fatal(err) - } - - if wc.ActivePath != sharedPath { - t.Errorf("ActivePath = %q, want %q", wc.ActivePath, sharedPath) - } - if wc.Scope != ScopeDelegate { - t.Errorf("Scope = %q, want delegate", wc.Scope) - } - if len(wc.ReadOnlyPaths) != 1 || wc.ReadOnlyPaths[0] != exportPath { - t.Errorf("ReadOnlyPaths = %v", wc.ReadOnlyPaths) - } - assertDirExists(t, wc.ActivePath) -} - -func TestResolve_DelegationEscapesBaseDir(t *testing.T) { - base := t.TempDir() - r := NewResolver() - _, err := r.Resolve(context.Background(), ResolveParams{ - AgentID: "agent-1", - UserID: "user-1", - BaseDir: base, - DelegateCtx: &DelegateContext{ - SharedPath: "/etc/shadow", - }, - }) - if err == nil { - t.Error("expected error for delegate path escaping base dir") - } -} - func TestResolve_EnforcementLabel(t *testing.T) { tests := []struct { name string diff --git a/internal/workspace/workspace_context.go b/internal/workspace/workspace_context.go index eaf5bdc0..f5224924 100644 --- a/internal/workspace/workspace_context.go +++ b/internal/workspace/workspace_context.go @@ -10,9 +10,9 @@ import "context" type Scope string const ( - ScopePersonal Scope = "personal" // single user, isolated - ScopeTeam Scope = "team" // team context, shared or isolated - ScopeDelegate Scope = "delegate" // delegated task, scoped access + ScopePersonal Scope = "personal" // single user, isolated + ScopeTeam Scope = "team" // team context, shared or isolated + ScopeDelegate Scope = "delegate" // delegated task, scoped access ) // WorkspaceContext is resolved ONCE at run start, immutable for the entire run. @@ -24,13 +24,6 @@ type WorkspaceContext struct { // Scope describes the access boundary type. Scope Scope - // ReadOnlyPaths are additional paths the agent can read but NOT write. - ReadOnlyPaths []string - - // SharedPath is the shared delegate area (read/write by both delegator + delegatee). - // nil when not in delegation context. - SharedPath *string - // TeamPath is the team workspace root (nil if not in team context). TeamPath *string @@ -65,7 +58,6 @@ type ResolveParams struct { PeerKind string // "direct" | "group" TeamID *string TeamConfig *TeamWorkspaceConfig - DelegateCtx *DelegateContext BaseDir string } @@ -81,13 +73,6 @@ func (c *TeamWorkspaceConfig) IsShared() bool { return c != nil && c.WorkspaceScope == "shared" } -// DelegateContext carries delegation-specific workspace overrides. -type DelegateContext struct { - LinkID string - SharedPath string - ExportPaths []string // read-only exports from delegator -} - // DefaultEnforcementLabel returns a human-readable workspace description // for system prompt injection based on scope and sharing mode. func DefaultEnforcementLabel(scope Scope, shared bool) string { diff --git a/migrations/000096_subagent_tasks_root_agent_scope.down.sql b/migrations/000096_subagent_tasks_root_agent_scope.down.sql new file mode 100644 index 00000000..2b23f2f6 --- /dev/null +++ b/migrations/000096_subagent_tasks_root_agent_scope.down.sql @@ -0,0 +1,9 @@ +DROP INDEX IF EXISTS idx_subagent_tasks_root_archive; +DROP INDEX IF EXISTS idx_subagent_tasks_root_session; +DROP INDEX IF EXISTS idx_subagent_tasks_root_status; + +ALTER TABLE subagent_tasks + DROP CONSTRAINT IF EXISTS fk_subagent_tasks_root_agent; + +ALTER TABLE subagent_tasks + DROP COLUMN IF EXISTS root_agent_id; diff --git a/migrations/000096_subagent_tasks_root_agent_scope.up.sql b/migrations/000096_subagent_tasks_root_agent_scope.up.sql new file mode 100644 index 00000000..292c5408 --- /dev/null +++ b/migrations/000096_subagent_tasks_root_agent_scope.up.sql @@ -0,0 +1,63 @@ +-- Persist immutable root-agent ownership separately from the human-readable key. +-- Existing rows prefer the UUID captured in metadata. Rows without that metadata +-- are backfilled by key only when exactly one matching agent predates the task. +-- This prevents a newly recreated same-key agent from inheriting older tasks. +-- Ambiguous or unmatched rows remain NULL and are intentionally inaccessible. +ALTER TABLE subagent_tasks + ADD COLUMN root_agent_id UUID; + +WITH metadata_owners AS ( + SELECT task.id AS task_id, agent.id AS root_agent_id + FROM subagent_tasks AS task + JOIN agents AS agent + ON agent.tenant_id = task.tenant_id + AND agent.id::text = task.metadata->>'root_agent_id' + WHERE task.metadata ? 'root_agent_id' +) +UPDATE subagent_tasks AS task +SET root_agent_id = owner.root_agent_id +FROM metadata_owners AS owner +WHERE task.id = owner.task_id; + +WITH unique_key_owners AS ( + SELECT task.id AS task_id, agent.id AS root_agent_id + FROM subagent_tasks AS task + JOIN agents AS agent + ON agent.tenant_id = task.tenant_id + AND agent.agent_key = task.parent_agent_key + AND agent.created_at < task.created_at + WHERE task.root_agent_id IS NULL + AND NOT (task.metadata ? 'root_agent_id') + AND NOT EXISTS ( + SELECT 1 + FROM agents AS other + WHERE other.tenant_id = agent.tenant_id + AND other.agent_key = agent.agent_key + AND other.created_at < task.created_at + AND other.id <> agent.id + ) +) +UPDATE subagent_tasks AS task +SET root_agent_id = owner.root_agent_id +FROM unique_key_owners AS owner +WHERE task.id = owner.task_id; + +ALTER TABLE subagent_tasks + ADD CONSTRAINT fk_subagent_tasks_root_agent + FOREIGN KEY (root_agent_id, tenant_id) + REFERENCES agents(id, tenant_id) + ON DELETE SET NULL (root_agent_id); + +CREATE INDEX idx_subagent_tasks_root_status + ON subagent_tasks(tenant_id, root_agent_id, status, created_at DESC) + WHERE root_agent_id IS NOT NULL; + +CREATE INDEX idx_subagent_tasks_root_session + ON subagent_tasks(tenant_id, root_agent_id, session_key, created_at DESC) + WHERE root_agent_id IS NOT NULL AND session_key IS NOT NULL; + +CREATE INDEX idx_subagent_tasks_root_archive + ON subagent_tasks(tenant_id, root_agent_id, completed_at, id) + WHERE root_agent_id IS NOT NULL + AND status IN ('completed', 'failed', 'cancelled') + AND archived_at IS NULL; diff --git a/skills/workspace-organizing/SKILL.md b/skills/workspace-organizing/SKILL.md index ac4ab107..273a793f 100644 --- a/skills/workspace-organizing/SKILL.md +++ b/skills/workspace-organizing/SKILL.md @@ -16,7 +16,7 @@ Keep agent workspaces tidy, predictable, and collision-free — and discoverable This skill governs file/folder layout and discovery inside: - `ActivePath` — primary read/write root for the current run -- `SharedPath` — delegate exchange area (when delegating) +- `inputs/` — read-only staged input alias available during an Agent Link artifact run - `TeamPath` — team workspace root (when in a team context) - **Vault** — the cross-workspace knowledge index (personal / team / shared scopes); searched via `vault_search`, read via `vault_read` @@ -173,12 +173,13 @@ Never set `deliver=true` on a `tmp/` file or unpromoted draft. - **Never overwrite a file in another teammate's namespace** (`shared//...`). Read is fine; write/move/delete is not - Before writing in the team root, list the directory and run `vault_search` first -### Delegate workspace (`Scope = delegate`, with `SharedPath`) +### Delegate workspace (`Scope = delegate`, isolated Agent Link exchange) -- **Inputs from delegator** live in `SharedPath/inputs/` — read-only unless told otherwise -- **Outputs back to delegator** go in `SharedPath/outputs/` -- Working notes the delegatee needs only for itself stay in `ActivePath` -- End each delegated task with `SharedPath/outputs/SUMMARY.md` describing what was produced and where +- **Inputs from delegator** are readable only through logical `inputs/...` paths. +- `ActivePath` is the logical `outputs/` root. Write output files using ordinary relative paths such as `report.md`; do not prefix them with `outputs/`. +- Files are not visible to the caller until the run and every synchronous nested child finish, then runtime validation publishes them atomically. +- After success, the caller can revisit the durable publication at `.delegations//`, including `manifest.json` and logical `outputs/...` entries. +- End each delegated task with `SUMMARY.md` describing what was produced and where. The runtime publishes it as `outputs/SUMMARY.md`. ## Workflow: Before Every File Write in a Shared Workspace diff --git a/tests/integration/sqlite_smoke_test.go b/tests/integration/sqlite_smoke_test.go index 69ac8148..7b19006d 100644 --- a/tests/integration/sqlite_smoke_test.go +++ b/tests/integration/sqlite_smoke_test.go @@ -91,6 +91,7 @@ func TestSQLiteSmokeTest(t *testing.T) { taskID := uuid.Must(uuid.NewV7()) task := &store.SubagentTaskData{ ParentAgentKey: agentKey, + RootAgentID: agentID, Subject: "smoke task", Description: "test", Status: "running", @@ -102,7 +103,7 @@ func TestSQLiteSmokeTest(t *testing.T) { if err := stores.SubagentTasks.Create(ctx, task); err != nil { t.Fatalf("Create: %v", err) } - got, err := stores.SubagentTasks.Get(ctx, taskID) + got, err := stores.SubagentTasks.Get(ctx, agentID, taskID) if err != nil { t.Fatalf("Get: %v", err) } diff --git a/tests/integration/v3_agent_links_store_test.go b/tests/integration/v3_agent_links_store_test.go index 475c78dc..dc661767 100644 --- a/tests/integration/v3_agent_links_store_test.go +++ b/tests/integration/v3_agent_links_store_test.go @@ -3,8 +3,9 @@ package integration import ( - "errors" + "context" "database/sql" + "errors" "testing" "github.com/google/uuid" @@ -266,6 +267,47 @@ func TestStoreAgentLink_GetLinkBetween(t *testing.T) { } } +func TestStoreAgentLink_GetLinkBetweenContract(t *testing.T) { + db := testDB(t) + tenantID, agentA := seedTenantAgent(t, db) + _, agentB := seedTenantAgent(t, db) + _, agentC := seedTenantAgent(t, db) + ctx := tenantCtx(tenantID) + ls := newLinkStore(db) + + reverse := makeLink(agentB, agentA, store.LinkDirectionInbound, store.LinkStatusActive) + direct := makeLink(agentA, agentB, store.LinkDirectionOutbound, store.LinkStatusActive) + for _, link := range []*store.AgentLinkData{reverse, direct} { + if err := ls.CreateLink(ctx, link); err != nil { + t.Fatalf("CreateLink: %v", err) + } + linkID := link.ID + t.Cleanup(func() { _, _ = db.Exec("DELETE FROM agent_links WHERE id = $1", linkID) }) + } + + got, err := ls.GetLinkBetween(ctx, agentA, agentB) + if err != nil { + t.Fatalf("GetLinkBetween(deterministic): %v", err) + } + if got == nil || got.ID != direct.ID { + t.Fatalf("GetLinkBetween chose %#v, want delegator-authored %s", got, direct.ID) + } + + missing, err := ls.GetLinkBetween(ctx, agentA, agentC) + if err != nil { + t.Fatalf("GetLinkBetween(not found): %v", err) + } + if missing != nil { + t.Fatalf("GetLinkBetween(not found) = %#v, want nil", missing) + } + + cancelled, cancel := context.WithCancel(ctx) + cancel() + if _, err := ls.GetLinkBetween(cancelled, agentA, agentB); !errors.Is(err, context.Canceled) { + t.Fatalf("GetLinkBetween(cancelled) error = %v, want context.Canceled", err) + } +} + func TestStoreAgentLink_TenantIsolation(t *testing.T) { db := testDB(t) tenantA, agentA1 := seedTenantAgent(t, db) @@ -317,13 +359,17 @@ func TestStoreAgentLink_DelegateTargets(t *testing.T) { tenantID, agentA := seedTenantAgent(t, db) _, agentB := seedTenantAgent(t, db) _, agentC := seedTenantAgent(t, db) + _, agentD := seedTenantAgent(t, db) ctx := tenantCtx(tenantID) ls := newLinkStore(db) - // Active outbound A→B, active outbound A→C. + // A→B outbound: A delegates to B. + // A→C inbound: C delegates to A. + // A↔D bidirectional: both delegate to each other. lAB := makeLink(agentA, agentB, store.LinkDirectionOutbound, store.LinkStatusActive) - lAC := makeLink(agentA, agentC, store.LinkDirectionOutbound, store.LinkStatusActive) - for _, l := range []*store.AgentLinkData{lAB, lAC} { + lAC := makeLink(agentA, agentC, store.LinkDirectionInbound, store.LinkStatusActive) + lAD := makeLink(agentA, agentD, store.LinkDirectionBidirectional, store.LinkStatusActive) + for _, l := range []*store.AgentLinkData{lAB, lAC, lAD} { if err := ls.CreateLink(ctx, l); err != nil { t.Fatalf("CreateLink: %v", err) } @@ -331,20 +377,48 @@ func TestStoreAgentLink_DelegateTargets(t *testing.T) { t.Cleanup(func() { db.Exec("DELETE FROM agent_links WHERE id = $1", lAB.ID) db.Exec("DELETE FROM agent_links WHERE id = $1", lAC.ID) + db.Exec("DELETE FROM agent_links WHERE id = $1", lAD.ID) }) - targets, err := ls.DelegateTargets(ctx, agentA) - if err != nil { - t.Fatalf("DelegateTargets: %v", err) + assertTargets := func(from uuid.UUID, want ...uuid.UUID) { + t.Helper() + targets, err := ls.DelegateTargets(ctx, from) + if err != nil { + t.Fatalf("DelegateTargets(%s): %v", from, err) + } + got := make(map[uuid.UUID]bool, len(targets)) + for _, link := range targets { + got[link.ID] = true + } + for _, id := range want { + if !got[id] { + t.Errorf("DelegateTargets(%s): link %s not found", from, id) + } + } + if len(got) != len(want) { + t.Errorf("DelegateTargets(%s): got %d links, want %d", from, len(got), len(want)) + } } - found := map[uuid.UUID]bool{} - for _, l := range targets { - found[l.ID] = true + + assertTargets(agentA, lAB.ID, lAD.ID) + assertTargets(agentB) + assertTargets(agentC, lAC.ID) + assertTargets(agentD, lAD.ID) + + if err := ls.UpdateLink(ctx, lAB.ID, map[string]any{"status": store.LinkStatusDisabled}); err != nil { + t.Fatalf("disable outbound link: %v", err) } - if !found[lAB.ID] { - t.Error("DelegateTargets: lAB not found") + assertTargets(agentA, lAD.ID) + + if err := ls.UpdateLink(ctx, lAC.ID, map[string]any{"direction": store.LinkDirectionOutbound}); err != nil { + t.Fatalf("change inbound direction: %v", err) } - if !found[lAC.ID] { - t.Error("DelegateTargets: lAC not found") + assertTargets(agentA, lAC.ID, lAD.ID) + assertTargets(agentC) + + if err := ls.DeleteLink(ctx, lAD.ID); err != nil { + t.Fatalf("delete bidirectional link: %v", err) } + assertTargets(agentA, lAC.ID) + assertTargets(agentD) } diff --git a/ui/desktop/frontend/src/components/agents/subagents-section.tsx b/ui/desktop/frontend/src/components/agents/subagents-section.tsx index 0a8a663f..c4f28fe3 100644 --- a/ui/desktop/frontend/src/components/agents/subagents-section.tsx +++ b/ui/desktop/frontend/src/components/agents/subagents-section.tsx @@ -25,12 +25,12 @@ export function SubagentsSection({ enabled, value, onToggle, onChange }: Subagen
update({ maxConcurrent: Math.min(numOrUndef(e.target.value) ?? 2, 2) })} + type="number" min={1} + value={value.maxConcurrent ?? 20} + onChange={(e) => update({ maxConcurrent: numOrUndef(e.target.value) ?? 20 })} className="w-full bg-surface-tertiary border border-border rounded-lg px-3 py-2 text-base md:text-sm text-text-primary focus:outline-none focus:ring-1 focus:ring-accent" /> -

Lite limit: 2

+

{t('configSections.subagents.maxConcurrentLiteHint')}

diff --git a/ui/desktop/frontend/src/i18n/locales/en/agents.json b/ui/desktop/frontend/src/i18n/locales/en/agents.json index 7b04a494..56e8e5fd 100644 --- a/ui/desktop/frontend/src/i18n/locales/en/agents.json +++ b/ui/desktop/frontend/src/i18n/locales/en/agents.json @@ -340,6 +340,7 @@ "title": "Subagents", "description": "Controls sub-agent spawning limits and behavior", "maxConcurrent": "Max Concurrent", + "maxConcurrentLiteHint": "Per-agent setting; Lite process safety cap: 2", "maxSpawnDepth": "Max Spawn Depth", "maxChildrenPerAgent": "Max Children Per Agent", "archiveAfter": "Archive After (minutes)", diff --git a/ui/desktop/frontend/src/i18n/locales/ru/agents.json b/ui/desktop/frontend/src/i18n/locales/ru/agents.json index e5972e41..6499f4e6 100644 --- a/ui/desktop/frontend/src/i18n/locales/ru/agents.json +++ b/ui/desktop/frontend/src/i18n/locales/ru/agents.json @@ -340,6 +340,7 @@ "title": "Субагенты", "description": "Управляет лимитами и поведением запуска субагентов", "maxConcurrent": "Макс. одновременно", + "maxConcurrentLiteHint": "Настройка для агента; защитный лимит процесса Lite: 2", "maxSpawnDepth": "Макс. глубина запуска", "maxChildrenPerAgent": "Макс. дочерних на агента", "archiveAfter": "Архивировать через (минуты)", diff --git a/ui/desktop/frontend/src/i18n/locales/vi/agents.json b/ui/desktop/frontend/src/i18n/locales/vi/agents.json index 39f5f39c..6f948a2e 100644 --- a/ui/desktop/frontend/src/i18n/locales/vi/agents.json +++ b/ui/desktop/frontend/src/i18n/locales/vi/agents.json @@ -340,6 +340,7 @@ "title": "Agent con", "description": "Kiểm soát giới hạn sinh sản agent con và hành vi", "maxConcurrent": "Tối đa đồng thời", + "maxConcurrentLiteHint": "Cài đặt theo agent; giới hạn an toàn toàn tiến trình của Lite: 2", "maxSpawnDepth": "Độ sâu sinh sản tối đa", "maxChildrenPerAgent": "Con tối đa mỗi agent", "archiveAfter": "Lưu trữ sau (phút)", diff --git a/ui/desktop/frontend/src/i18n/locales/zh/agents.json b/ui/desktop/frontend/src/i18n/locales/zh/agents.json index 04c0424b..f25ebc87 100644 --- a/ui/desktop/frontend/src/i18n/locales/zh/agents.json +++ b/ui/desktop/frontend/src/i18n/locales/zh/agents.json @@ -340,6 +340,7 @@ "title": "子Agent", "description": "控制子Agent生成限制和行为", "maxConcurrent": "最大并发数", + "maxConcurrentLiteHint": "每个 Agent 的设置；Lite 进程安全上限：2", "maxSpawnDepth": "最大生成深度", "maxChildrenPerAgent": "每Agent最大子数", "archiveAfter": "归档时间（分钟）", diff --git a/ui/web/src/i18n/locales/en/agents.json b/ui/web/src/i18n/locales/en/agents.json index e8b8dec0..544ba356 100644 --- a/ui/web/src/i18n/locales/en/agents.json +++ b/ui/web/src/i18n/locales/en/agents.json @@ -750,6 +750,7 @@ "description": "Controls sub-agent spawning limits and behavior", "maxConcurrent": "Max Concurrent", "maxSpawnDepth": "Max Spawn Depth", + "maxSpawnDepthTip": "Maximum depth of each agent's own sub-agent spawn tree. Agent Link delegation starts a new tree governed by the target agent's settings. At depth 1, an agent can spawn direct sub-agents, but those sub-agents cannot spawn again.", "maxChildrenPerAgent": "Max Children Per Agent", "archiveAfter": "Archive After (minutes)", "maxRetries": "Max Retries", diff --git a/ui/web/src/i18n/locales/en/config.json b/ui/web/src/i18n/locales/en/config.json index c813a14f..98d290be 100644 --- a/ui/web/src/i18n/locales/en/config.json +++ b/ui/web/src/i18n/locales/en/config.json @@ -94,7 +94,7 @@ "agents.subagents.maxConcurrent": "Max Concurrent", "agents.subagents.maxConcurrentTip": "Maximum number of sub-agents that can run simultaneously.", "agents.subagents.maxSpawnDepth": "Max Spawn Depth", - "agents.subagents.maxSpawnDepthTip": "Maximum nesting depth for sub-agent chains.", + "agents.subagents.maxSpawnDepthTip": "Maximum depth of each agent's own sub-agent spawn tree. Agent Link delegation starts a new tree governed by the target agent's settings. At depth 1, an agent can spawn direct sub-agents, but those sub-agents cannot spawn again.", "agents.subagents.maxChildrenPerAgent": "Max Children Per Agent", "agents.subagents.maxChildrenPerAgentTip": "Maximum number of child sub-agents a single agent can spawn.", "agents.subagents.archiveAfterMin": "Archive After (min)", diff --git a/ui/web/src/i18n/locales/en/teams.json b/ui/web/src/i18n/locales/en/teams.json index 8d717e51..08fbc844 100644 --- a/ui/web/src/i18n/locales/en/teams.json +++ b/ui/web/src/i18n/locales/en/teams.json @@ -394,6 +394,7 @@ "description": "Description", "descriptionPlaceholder": "Optional description...", "maxConcurrent": "Max Concurrent", + "maxConcurrentHint": "Compatibility metadata only; Agent Link runtime admission uses the process safety cap and bounded pending queue.", "status": "Status", "active": "Active", "disabled": "Disabled", diff --git a/ui/web/src/i18n/locales/ko/agents.json b/ui/web/src/i18n/locales/ko/agents.json index 1c90047a..1524d50e 100644 --- a/ui/web/src/i18n/locales/ko/agents.json +++ b/ui/web/src/i18n/locales/ko/agents.json @@ -741,6 +741,7 @@ "description": "서브에이전트 생성 제한 및 동작 제어", "maxConcurrent": "최대 동시", "maxSpawnDepth": "최대 생성 깊이", + "maxSpawnDepthTip": "각 에이전트가 직접 생성하는 서브에이전트 트리의 최대 깊이입니다. Agent Link 위임은 대상 에이전트의 설정을 사용하는 새 트리를 시작합니다. 깊이가 1이면 직접 서브에이전트를 생성할 수 있지만 해당 서브에이전트는 다시 생성할 수 없습니다.", "maxChildrenPerAgent": "에이전트당 최대 자식", "archiveAfter": "보관 시간 (분)", "modelOverride": "모델 재정의", diff --git a/ui/web/src/i18n/locales/ko/config.json b/ui/web/src/i18n/locales/ko/config.json index ec7e2a15..72c9a020 100644 --- a/ui/web/src/i18n/locales/ko/config.json +++ b/ui/web/src/i18n/locales/ko/config.json @@ -96,7 +96,7 @@ "agents.subagents.maxConcurrent": "최대 동시", "agents.subagents.maxConcurrentTip": "동시에 실행할 수 있는 서브에이전트의 최대 수입니다.", "agents.subagents.maxSpawnDepth": "최대 생성 깊이", - "agents.subagents.maxSpawnDepthTip": "서브에이전트 체인의 최대 중첩 깊이입니다.", + "agents.subagents.maxSpawnDepthTip": "각 에이전트가 직접 생성하는 서브에이전트 트리의 최대 깊이입니다. Agent Link 위임은 대상 에이전트의 설정을 사용하는 새 트리를 시작합니다. 깊이가 1이면 직접 서브에이전트를 생성할 수 있지만 해당 서브에이전트는 다시 생성할 수 없습니다.", "agents.subagents.maxChildrenPerAgent": "에이전트당 최대 자식", "agents.subagents.maxChildrenPerAgentTip": "단일 에이전트가 생성할 수 있는 최대 자식 서브에이전트 수입니다.", "agents.subagents.archiveAfterMin": "보관 시간 (분)", diff --git a/ui/web/src/i18n/locales/ko/teams.json b/ui/web/src/i18n/locales/ko/teams.json index 95897e1e..b3d1b29b 100644 --- a/ui/web/src/i18n/locales/ko/teams.json +++ b/ui/web/src/i18n/locales/ko/teams.json @@ -386,6 +386,7 @@ "description": "설명", "descriptionPlaceholder": "선택적 설명...", "maxConcurrent": "최대 동시", + "maxConcurrentHint": "호환성 메타데이터로만 유지됩니다. Agent Link 런타임 승인은 프로세스 안전 한도와 제한된 대기열을 사용합니다.", "status": "상태", "active": "활성", "disabled": "비활성화됨", diff --git a/ui/web/src/i18n/locales/ru/agents.json b/ui/web/src/i18n/locales/ru/agents.json index 7ea5f853..6bb8bb2e 100644 --- a/ui/web/src/i18n/locales/ru/agents.json +++ b/ui/web/src/i18n/locales/ru/agents.json @@ -750,6 +750,7 @@ "description": "Управляет лимитами и поведением порождения субагентов", "maxConcurrent": "Макс. одновременно", "maxSpawnDepth": "Макс. глубина порождения", + "maxSpawnDepthTip": "Максимальная глубина собственного дерева субагентов каждого агента. Делегирование через Agent Link начинает новое дерево с настройками целевого агента. При глубине 1 агент может запустить прямых субагентов, но они не смогут запускать новых.", "maxChildrenPerAgent": "Макс. дочерних на агента", "archiveAfter": "Архивировать через (минут)", "maxRetries": "Макс. повторов", diff --git a/ui/web/src/i18n/locales/ru/config.json b/ui/web/src/i18n/locales/ru/config.json index ccaad697..c331d9ab 100644 --- a/ui/web/src/i18n/locales/ru/config.json +++ b/ui/web/src/i18n/locales/ru/config.json @@ -99,7 +99,7 @@ "agents.subagents.maxConcurrent": "Макс. одновременно", "agents.subagents.maxConcurrentTip": "Максимальное количество субагентов, которые могут работать одновременно.", "agents.subagents.maxSpawnDepth": "Макс. глубина порождения", - "agents.subagents.maxSpawnDepthTip": "Максимальная глубина вложенности для цепочек субагентов.", + "agents.subagents.maxSpawnDepthTip": "Максимальная глубина собственного дерева субагентов каждого агента. Делегирование через Agent Link начинает новое дерево с настройками целевого агента. При глубине 1 агент может запустить прямых субагентов, но они не смогут запускать новых.", "agents.subagents.maxChildrenPerAgent": "Макс. дочерних на агента", "agents.subagents.maxChildrenPerAgentTip": "Максимальное количество дочерних субагентов, которое может породить один агент.", "agents.subagents.archiveAfterMin": "Архивировать через (мин)", diff --git a/ui/web/src/i18n/locales/ru/teams.json b/ui/web/src/i18n/locales/ru/teams.json index e29b17bb..273d8768 100644 --- a/ui/web/src/i18n/locales/ru/teams.json +++ b/ui/web/src/i18n/locales/ru/teams.json @@ -394,6 +394,7 @@ "description": "Описание", "descriptionPlaceholder": "Необязательное описание...", "maxConcurrent": "Макс. одновременных", + "maxConcurrentHint": "Поле сохранено только как метаданные совместимости; Agent Link использует защитный лимит процесса и ограниченную очередь ожидания.", "status": "Статус", "active": "Активна", "disabled": "Отключена", diff --git a/ui/web/src/i18n/locales/vi/agents.json b/ui/web/src/i18n/locales/vi/agents.json index 7797cca3..43fc70c8 100644 --- a/ui/web/src/i18n/locales/vi/agents.json +++ b/ui/web/src/i18n/locales/vi/agents.json @@ -735,6 +735,7 @@ "description": "Kiểm soát giới hạn sinh sản agent con và hành vi", "maxConcurrent": "Tối đa đồng thời", "maxSpawnDepth": "Độ sâu sinh sản tối đa", + "maxSpawnDepthTip": "Độ sâu tối đa của cây sub-agent do từng agent tự spawn. Delegate qua Agent Link bắt đầu một cây mới và dùng setting của agent đích. Với depth 1, agent có thể spawn sub-agent trực tiếp, nhưng các sub-agent đó không thể spawn tiếp.", "maxChildrenPerAgent": "Con tối đa mỗi agent", "archiveAfter": "Lưu trữ sau (phút)", "maxRetries": "Số lần thử lại tối đa", diff --git a/ui/web/src/i18n/locales/vi/config.json b/ui/web/src/i18n/locales/vi/config.json index 0da57655..d4f64547 100644 --- a/ui/web/src/i18n/locales/vi/config.json +++ b/ui/web/src/i18n/locales/vi/config.json @@ -94,7 +94,7 @@ "agents.subagents.maxConcurrent": "Tối đa đồng thời", "agents.subagents.maxConcurrentTip": "Số subagent tối đa có thể chạy đồng thời.", "agents.subagents.maxSpawnDepth": "Độ sâu tạo tối đa", - "agents.subagents.maxSpawnDepthTip": "Độ sâu lồng nhau tối đa cho chuỗi subagent.", + "agents.subagents.maxSpawnDepthTip": "Độ sâu tối đa của cây sub-agent do từng agent tự spawn. Delegate qua Agent Link bắt đầu một cây mới và dùng setting của agent đích. Với depth 1, agent có thể spawn sub-agent trực tiếp, nhưng các sub-agent đó không thể spawn tiếp.", "agents.subagents.maxChildrenPerAgent": "Subagent tối đa mỗi agent", "agents.subagents.maxChildrenPerAgentTip": "Số subagent tối đa mà một agent có thể tạo.", "agents.subagents.archiveAfterMin": "Lưu trữ sau (phút)", diff --git a/ui/web/src/i18n/locales/vi/teams.json b/ui/web/src/i18n/locales/vi/teams.json index 348f067c..9eaeb63a 100644 --- a/ui/web/src/i18n/locales/vi/teams.json +++ b/ui/web/src/i18n/locales/vi/teams.json @@ -394,6 +394,7 @@ "description": "Mô tả", "descriptionPlaceholder": "Mô tả tùy chọn...", "maxConcurrent": "Tối đa đồng thời", + "maxConcurrentHint": "Chỉ là metadata tương thích; Agent Link dùng giới hạn an toàn toàn tiến trình và hàng đợi có giới hạn.", "status": "Trạng thái", "active": "Hoạt động", "disabled": "Tắt", diff --git a/ui/web/src/i18n/locales/zh/agents.json b/ui/web/src/i18n/locales/zh/agents.json index e65e076c..aaa758df 100644 --- a/ui/web/src/i18n/locales/zh/agents.json +++ b/ui/web/src/i18n/locales/zh/agents.json @@ -735,6 +735,7 @@ "description": "控制子Agent生成限制和行为", "maxConcurrent": "最大并发数", "maxSpawnDepth": "最大生成深度", + "maxSpawnDepthTip": "每个 Agent 自己生成的子 Agent 树的最大深度。通过 Agent Link 委派会开始一棵新树，并使用目标 Agent 的设置。深度为 1 时，Agent 可以生成直接子 Agent，但这些子 Agent 不能继续生成。", "maxChildrenPerAgent": "每Agent最大子数", "archiveAfter": "归档时间（分钟）", "maxRetries": "最大重试次数", diff --git a/ui/web/src/i18n/locales/zh/config.json b/ui/web/src/i18n/locales/zh/config.json index a1b7a9ca..0bc41351 100644 --- a/ui/web/src/i18n/locales/zh/config.json +++ b/ui/web/src/i18n/locales/zh/config.json @@ -94,7 +94,7 @@ "agents.subagents.maxConcurrent": "最大并发数", "agents.subagents.maxConcurrentTip": "可同时运行的最大子 Agent 数量。", "agents.subagents.maxSpawnDepth": "最大生成深度", - "agents.subagents.maxSpawnDepthTip": "子 Agent 链的最大嵌套深度。", + "agents.subagents.maxSpawnDepthTip": "每个 Agent 自己生成的子 Agent 树的最大深度。通过 Agent Link 委派会开始一棵新树，并使用目标 Agent 的设置。深度为 1 时，Agent 可以生成直接子 Agent，但这些子 Agent 不能继续生成。", "agents.subagents.maxChildrenPerAgent": "每 Agent 最大子数", "agents.subagents.maxChildrenPerAgentTip": "单个 Agent 可生成的最大子 Agent 数量。", "agents.subagents.archiveAfterMin": "归档时间（分钟）", diff --git a/ui/web/src/i18n/locales/zh/teams.json b/ui/web/src/i18n/locales/zh/teams.json index d7b064cd..d260355e 100644 --- a/ui/web/src/i18n/locales/zh/teams.json +++ b/ui/web/src/i18n/locales/zh/teams.json @@ -394,6 +394,7 @@ "description": "描述", "descriptionPlaceholder": "可选描述...", "maxConcurrent": "最大并发数", + "maxConcurrentHint": "仅保留为兼容元数据；Agent Link 运行时准入使用进程安全上限和有界等待队列。", "status": "状态", "active": "活跃", "disabled": "禁用", diff --git a/ui/web/src/pages/agents/agent-detail/config-sections/subagents-section.tsx b/ui/web/src/pages/agents/agent-detail/config-sections/subagents-section.tsx index 157e101c..999dadc2 100644 --- a/ui/web/src/pages/agents/agent-detail/config-sections/subagents-section.tsx +++ b/ui/web/src/pages/agents/agent-detail/config-sections/subagents-section.tsx @@ -49,7 +49,7 @@ export function SubagentsSection({ enabled, value, onToggle, onChange }: Subagen />
- {t(`${s}.maxSpawnDepth`)} + {t(`${s}.maxSpawnDepth`)}