diff --git a/cmd/gateway.go b/cmd/gateway.go index b78f955c..e7805c42 100644 --- a/cmd/gateway.go +++ b/cmd/gateway.go @@ -181,17 +181,16 @@ func runGateway() { } setupMemoryEmbeddings(pgStores, providerRegistry) + // Resolve background provider for consolidation + vault enrichment. + // Fallback: background.provider → agent.default_provider → first registered provider. + bgProvider, bgModel := resolveBackgroundProvider(cfg, providerRegistry) + // V3: Wire consolidation pipeline (episodic → semantic → KG → dreaming) if pgStores.Episodic != nil { - var consolidationProvider providers.Provider - if names := providerRegistry.ListForTenant(providers.MasterTenantID); len(names) > 0 { - consolidationProvider, _ = providerRegistry.GetForTenant(providers.MasterTenantID, names[0]) - } - if consolidationProvider != nil { - // Create KG extractor for semantic worker (entity/relation extraction from episodic summaries) + if bgProvider != nil { var kgExtractor *kg.Extractor if pgStores.KnowledgeGraph != nil { - kgExtractor = kg.NewExtractor(consolidationProvider, consolidationProvider.DefaultModel(), 0) + kgExtractor = kg.NewExtractor(bgProvider, bgModel, 0) } cleanupConsolidation := consolidation.Register(consolidation.ConsolidationDeps{ EpisodicStore: pgStores.Episodic, @@ -199,36 +198,28 @@ func runGateway() { KGStore: pgStores.KnowledgeGraph, SessionStore: pgStores.Sessions, EventBus: domainBus, - Provider: consolidationProvider, - Model: consolidationProvider.DefaultModel(), + Provider: bgProvider, + Model: bgModel, Extractor: kgExtractor, - // Per-agent dreaming overrides (MemoryConfig.Dreaming JSONB). - AgentStore: pgStores.Agents, + AgentStore: pgStores.Agents, }) defer cleanupConsolidation() - slog.Info("consolidation pipeline registered") + slog.Info("consolidation pipeline registered", "provider", bgProvider.Name(), "model", bgModel) } else { slog.Warn("consolidation pipeline skipped: no provider available") } } // V3: Wire vault enrichment worker (async summary + embedding + auto-linking). - // Resolves provider independently from consolidation pipeline. - if pgStores.Vault != nil { - var vaultProvider providers.Provider - if names := providerRegistry.ListForTenant(providers.MasterTenantID); len(names) > 0 { - vaultProvider, _ = providerRegistry.GetForTenant(providers.MasterTenantID, names[0]) - } - if vaultProvider != nil { - cleanupVaultEnrich := vault.RegisterEnrichWorker(vault.EnrichWorkerDeps{ - VaultStore: pgStores.Vault, - Provider: vaultProvider, - Model: vaultProvider.DefaultModel(), - EventBus: domainBus, - }) - defer cleanupVaultEnrich() - slog.Info("vault enrichment worker registered") - } + if pgStores.Vault != nil && bgProvider != nil { + cleanupVaultEnrich := vault.RegisterEnrichWorker(vault.EnrichWorkerDeps{ + VaultStore: pgStores.Vault, + Provider: bgProvider, + Model: bgModel, + EventBus: domainBus, + }) + defer cleanupVaultEnrich() + slog.Info("vault enrichment worker registered", "provider", bgProvider.Name(), "model", bgModel) } loadBootstrapFiles(pgStores, workspace, agentCfg) @@ -319,7 +310,11 @@ func runGateway() { // Wire dependencies for system prompt preview parity. if agentsH != nil { agentsH.SetPreviewDeps(toolsReg, skillsLoader) - agentsH.SetPreviewStores(pgStores.Teams, pgStores.AgentLinks) + var skillAccess store.SkillAccessStore + if pgStores.Skills != nil { + skillAccess, _ = pgStores.Skills.(store.SkillAccessStore) + } + agentsH.SetPreviewStores(pgStores.Teams, pgStores.AgentLinks, skillAccess) } // External wake/trigger API @@ -543,3 +538,39 @@ func runGateway() { sigCh: sigCh, }) } + +// resolveBackgroundProvider picks the LLM provider+model for background workers +// (vault enrichment, consolidation). Fallback chain: +// +// background.provider/model → agent.default_provider/model → first registered provider. +func resolveBackgroundProvider(cfg *config.Config, reg *providers.Registry) (providers.Provider, string) { + try := func(name, model string) (providers.Provider, string, bool) { + if name == "" { + return nil, "", false + } + p, err := reg.GetForTenant(providers.MasterTenantID, name) + if err != nil || p == nil { + return nil, "", false + } + if model == "" { + model = p.DefaultModel() + } + return p, model, true + } + + // 1. Explicit background config + if p, m, ok := try(cfg.Gateway.BackgroundProvider, cfg.Gateway.BackgroundModel); ok { + return p, m + } + // 2. Agent default provider + if p, m, ok := try(cfg.Agents.Defaults.Provider, cfg.Agents.Defaults.Model); ok { + return p, m + } + // 3. First registered provider (legacy fallback) + if names := reg.ListForTenant(providers.MasterTenantID); len(names) > 0 { + if p, m, ok := try(names[0], ""); ok { + return p, m + } + } + return nil, "" +} diff --git a/cmd/gateway_http_wiring.go b/cmd/gateway_http_wiring.go index 1bf0653a..d14942ff 100644 --- a/cmd/gateway_http_wiring.go +++ b/cmd/gateway_http_wiring.go @@ -175,7 +175,7 @@ func (d *gatewayDeps) wireHTTPHandlersOnServer( // V3: Knowledge Vault document API if d.pgStores != nil && d.pgStores.Vault != nil { - d.server.SetVaultHandler(httpapi.NewVaultHandler(d.pgStores.Vault, d.pgStores.Teams, d.workspace, d.domainBus)) + d.server.SetVaultHandler(httpapi.NewVaultHandler(d.pgStores.Vault, d.pgStores.Teams, d.workspace, d.domainBus, d.pgStores.Agents, d.pgStores.Teams)) } // V3: Episodic memory summaries API diff --git a/cmd/gateway_system_config_sync.go b/cmd/gateway_system_config_sync.go index 87068d3c..7e8b5e02 100644 --- a/cmd/gateway_system_config_sync.go +++ b/cmd/gateway_system_config_sync.go @@ -106,6 +106,10 @@ func seedConfigForContext(ctx context.Context, sc store.SystemConfigStore, cfg * setBool("gateway.tool_status", cfg.Gateway.ToolStatus) setInt("gateway.task_recovery_interval_sec", cfg.Gateway.TaskRecoveryIntervalSec) + // Background workers + set("background.provider", cfg.Gateway.BackgroundProvider) + set("background.model", cfg.Gateway.BackgroundModel) + // Tools set("tools.profile", cfg.Tools.Profile) setInt("tools.rate_limit_per_hour", cfg.Tools.RateLimitPerHour) diff --git a/internal/agent/preview_prompt.go b/internal/agent/preview_prompt.go index 35a4bdfe..abc71ece 100644 --- a/internal/agent/preview_prompt.go +++ b/internal/agent/preview_prompt.go @@ -4,34 +4,49 @@ import ( "context" "path/filepath" "slices" + "strings" "github.com/google/uuid" "github.com/nextlevelbuilder/goclaw/internal/bootstrap" "github.com/nextlevelbuilder/goclaw/internal/providers" "github.com/nextlevelbuilder/goclaw/internal/store" + "github.com/nextlevelbuilder/goclaw/internal/tokencount" + "github.com/nextlevelbuilder/goclaw/internal/tools" ) // PreviewDeps holds optional dependencies for building a preview system prompt. // All fields are nil-safe — missing deps simply skip resolution for that section. type PreviewDeps struct { - AgentStore store.AgentStore - TeamStore store.TeamStore - AgentLinks store.AgentLinkStore - ProviderReg *providers.Registry - ToolLister interface{ List() []string } - SkillsLoader interface { + AgentStore store.AgentStore + TeamStore store.TeamStore + AgentLinks store.AgentLinkStore + ProviderReg *providers.Registry + SkillAccessStore store.SkillAccessStore + ToolLister interface { + List() []string + Get(name string) (tools.Tool, bool) + Aliases() map[string]string + } + SkillsLoader interface { BuildPinnedSummary(ctx context.Context, names []string) string + BuildSummary(ctx context.Context, allowList []string) string } DataDir string // for team workspace path construction } +// PreviewResult holds the output of BuildPreviewPrompt. +type PreviewResult struct { + Prompt string + ToolDefs []providers.ToolDefinition // tool definitions (schemas) as sent to the LLM +} + // BuildPreviewPrompt builds a system prompt for preview purposes. // Reuses the same BuildSystemPrompt() as the LLM pipeline, resolving as many // fields as possible from agent data + DB stores. Runtime-only fields // (channel, peer kind, session context, credentials) are left at zero values — // BuildSystemPrompt already nil-checks every field. -func BuildPreviewPrompt(ctx context.Context, ag *store.AgentData, mode PromptMode, userID string, deps PreviewDeps) string { +func BuildPreviewPrompt(ctx context.Context, ag *store.AgentData, mode PromptMode, userID string, deps PreviewDeps) PreviewResult { // --- Context files --- var contextFiles []bootstrap.ContextFile if deps.AgentStore != nil { @@ -62,6 +77,63 @@ func BuildPreviewPrompt(ctx context.Context, ag *store.AgentData, mode PromptMod toolNames = fallbackPreviewToolNames } + // --- skill_manage gating (matches loop_history.go:124-131) --- + if !ag.ParseSkillEvolve() { + filtered := make([]string, 0, len(toolNames)) + for _, n := range toolNames { + if n != "skill_manage" { + filtered = append(filtered, n) + } + } + toolNames = filtered + } + + // --- Basic agent tool policy: deny-only (Allow/AlsoAllow/ByProvider not applied). + // Full PolicyEngine requires runtime state (provider name, channel) not available in preview. --- + if toolPolicy := ag.ParseToolsConfig(); toolPolicy != nil && len(toolPolicy.Deny) > 0 { + denySet := make(map[string]bool, len(toolPolicy.Deny)) + for _, d := range toolPolicy.Deny { + denySet[d] = true + } + filtered := make([]string, 0, len(toolNames)) + for _, n := range toolNames { + if !denySet[n] { + filtered = append(filtered, n) + } + } + toolNames = filtered + } + + // --- Alias exclusion (matches loop_history.go:136-146) --- + if deps.ToolLister != nil { + if aliasSet := deps.ToolLister.Aliases(); len(aliasSet) > 0 { + filtered := make([]string, 0, len(toolNames)) + for _, n := range toolNames { + if _, isAlias := aliasSet[n]; !isAlias { + filtered = append(filtered, n) + } + } + toolNames = filtered + } + } + + // --- MCP tool descriptions (matches loop_history_supplement.go:44-58) --- + var mcpToolDescs map[string]string + if deps.ToolLister != nil { + descs := make(map[string]string) + for _, name := range toolNames { + if !strings.HasPrefix(name, "mcp_") || name == "mcp_tool_search" { + continue + } + if tool, ok := deps.ToolLister.Get(name); ok { + descs[name] = tool.Description() + } + } + if len(descs) > 0 { + mcpToolDescs = descs + } + } + // --- Sandbox --- sandboxCfg := ag.ParseSandboxConfig() sandboxEnabled := sandboxCfg != nil && sandboxCfg.Mode != "" && sandboxCfg.Mode != "off" @@ -76,6 +148,32 @@ func BuildPreviewPrompt(ctx context.Context, ag *store.AgentData, mode PromptMod pinnedSummary = deps.SkillsLoader.BuildPinnedSummary(ctx, pinnedSkills) } + // --- Skills summary (BuildSummary + token count) --- + var skillsSummary string + if deps.SkillsLoader != nil { + var skillAllowList []string + if deps.SkillAccessStore != nil { + if accessible, err := deps.SkillAccessStore.ListAccessible(ctx, ag.ID, userID); err == nil { + skillAllowList = make([]string, 0, len(accessible)) + for _, sk := range accessible { + skillAllowList = append(skillAllowList, sk.Slug) + } + } else { + // On error: empty list (no skills). Preview is diagnostic; safer than showing all. + skillAllowList = []string{} + } + } + + summary := deps.SkillsLoader.BuildSummary(ctx, skillAllowList) + if summary != "" { + tokens := tokencount.NewFallbackCounter().Count("claude-3", summary) + if tokens <= skillInlineMaxTokens { + skillsSummary = summary + } + // Over threshold → search-only mode (skillsSummary stays empty) + } + } + // --- Provider contribution --- var providerContrib *providers.PromptContribution if deps.ProviderReg != nil && ag.Provider != "" { @@ -121,8 +219,31 @@ func BuildPreviewPrompt(ctx context.Context, ag *store.AgentData, mode PromptMod } } + // --- Tool definitions (schemas sent to LLM alongside the system prompt) --- + var toolDefs []providers.ToolDefinition + if deps.ToolLister != nil { + for _, name := range toolNames { + if tool, ok := deps.ToolLister.Get(name); ok { + toolDefs = append(toolDefs, tools.ToProviderDef(tool)) + } + } + // Include alias definitions (LLM receives both canonical + aliases) + for alias, canonical := range deps.ToolLister.Aliases() { + if tool, ok := deps.ToolLister.Get(canonical); ok { + toolDefs = append(toolDefs, providers.ToolDefinition{ + Type: "function", + Function: providers.ToolFunctionSchema{ + Name: alias, + Description: tool.Description(), + Parameters: tool.Parameters(), + }, + }) + } + } + } + // --- Build system prompt (same function as LLM pipeline) --- - return BuildSystemPrompt(SystemPromptConfig{ + prompt := BuildSystemPrompt(SystemPromptConfig{ AgentID: ag.AgentKey, AgentUUID: ag.ID.String(), DisplayName: ag.DisplayName, @@ -146,6 +267,8 @@ func BuildPreviewPrompt(ctx context.Context, ag *store.AgentData, mode PromptMod SandboxEnabled: sandboxEnabled, SandboxContainerDir: sandboxContainerDir, PinnedSkillsSummary: pinnedSummary, + SkillsSummary: skillsSummary, + MCPToolDescs: mcpToolDescs, IsTeamContext: isTeamCtx, TeamWorkspace: teamWorkspace, TeamMembers: teamMembers, @@ -154,8 +277,9 @@ func BuildPreviewPrompt(ctx context.Context, ag *store.AgentData, mode PromptMod OrchMode: orchMode, // Runtime-only fields left at zero: Channel, ChannelType, ChatTitle, // PeerKind, OwnerIDs, ExtraPrompt, CredentialCLIContext, IsBootstrap, - // MCPToolDescs, SkillsSummary, SandboxWorkspaceAccess + // SandboxWorkspaceAccess }) + return PreviewResult{Prompt: prompt, ToolDefs: toolDefs} } // mergePreviewUserFiles overlays per-user files onto base agent-level files. diff --git a/internal/agent/preview_prompt_test.go b/internal/agent/preview_prompt_test.go new file mode 100644 index 00000000..70177258 --- /dev/null +++ b/internal/agent/preview_prompt_test.go @@ -0,0 +1,238 @@ +package agent + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/google/uuid" + + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +func baseAgent() *store.AgentData { + return &store.AgentData{ + BaseModel: store.BaseModel{ID: uuid.New()}, + AgentKey: "test-agent", + AgentType: store.AgentTypePredefined, + Workspace: "/workspace", + } +} + +func TestBuildPreviewPrompt_NilDeps(t *testing.T) { + r := BuildPreviewPrompt(context.Background(), baseAgent(), PromptFull, "", PreviewDeps{}) + if r.Prompt == "" { + t.Fatal("expected non-empty prompt with nil deps") + } + if !strings.Contains(r.Prompt, "read_file") { + t.Error("expected fallback tool names in prompt") + } + // No tool lister → no tool defs + if len(r.ToolDefs) != 0 { + t.Errorf("expected no tool defs with nil ToolLister, got %d", len(r.ToolDefs)) + } +} + +func TestBuildPreviewPrompt_SkillsInline(t *testing.T) { + r := BuildPreviewPrompt(context.Background(), baseAgent(), PromptFull, "", PreviewDeps{ + SkillsLoader: &mockSkillsLoader{ + summary: "\nGit operations\n", + }, + }) + if !strings.Contains(r.Prompt, "") { + t.Error("expected skills XML inlined in prompt") + } +} + +func TestBuildPreviewPrompt_SkillsSearchMode(t *testing.T) { + bigSummary := strings.Repeat("x", 10000) + r := BuildPreviewPrompt(context.Background(), baseAgent(), PromptFull, "", PreviewDeps{ + SkillsLoader: &mockSkillsLoader{summary: bigSummary}, + }) + if strings.Contains(r.Prompt, bigSummary) { + t.Error("expected large summary to be excluded (search-only mode)") + } +} + +func TestBuildPreviewPrompt_PinnedSkillsHybrid(t *testing.T) { + ag := baseAgent() + ag.OtherConfig = []byte(`{"pinned_skills":["deploy"]}`) + r := BuildPreviewPrompt(context.Background(), ag, PromptFull, "", PreviewDeps{ + SkillsLoader: &mockSkillsLoader{ + pinned: "Deploy to prod", + summary: "\nGit ops\n", + }, + }) + if !strings.Contains(r.Prompt, "deploy") || !strings.Contains(r.Prompt, "Pinned skills") { + t.Error("expected pinned skills section in prompt") + } +} + +func TestBuildPreviewPrompt_SkillAllowList(t *testing.T) { + ag := baseAgent() + loader := &mockSkillsLoader{ + summary: "ok", + } + r := BuildPreviewPrompt(context.Background(), ag, PromptFull, "user1", PreviewDeps{ + SkillsLoader: loader, + SkillAccessStore: &mockSkillAccessStore{ + accessible: []store.SkillInfo{{Slug: "allowed-skill"}}, + }, + }) + if !strings.Contains(r.Prompt, "") { + t.Error("expected filtered skills in prompt") + } + if len(loader.capturedAllow) != 1 || loader.capturedAllow[0] != "allowed-skill" { + t.Errorf("expected allow list [allowed-skill], got %v", loader.capturedAllow) + } +} + +func TestBuildPreviewPrompt_SkillAccessStoreError(t *testing.T) { + ag := baseAgent() + loader := &mockSkillsLoader{ + summary: "desc", + } + r := BuildPreviewPrompt(context.Background(), ag, PromptFull, "user1", PreviewDeps{ + SkillsLoader: loader, + SkillAccessStore: &mockSkillAccessStore{err: errors.New("db error")}, + }) + if r.Prompt == "" { + t.Fatal("expected non-empty prompt on SkillAccessStore error") + } + if loader.capturedAllow == nil || len(loader.capturedAllow) != 0 { + t.Errorf("expected empty (non-nil) allow list on error, got %v", loader.capturedAllow) + } +} + +func TestBuildPreviewPrompt_MCPToolDescs(t *testing.T) { + r := BuildPreviewPrompt(context.Background(), baseAgent(), PromptFull, "", PreviewDeps{ + ToolLister: &mockToolLister{ + tools: map[string]string{ + "read_file": "Read a file", + "mcp_pg_query": "Run PostgreSQL queries", + }, + }, + }) + if !strings.Contains(r.Prompt, "mcp_pg_query") { + t.Error("expected MCP tool description in prompt") + } +} + +func TestBuildPreviewPrompt_MCPToolSearchExcluded(t *testing.T) { + r := BuildPreviewPrompt(context.Background(), baseAgent(), PromptFull, "", PreviewDeps{ + ToolLister: &mockToolLister{ + tools: map[string]string{ + "read_file": "Read a file", + "mcp_tool_search": "Search MCP tools", + }, + }, + }) + if strings.Contains(r.Prompt, "Search MCP tools") { + t.Error("mcp_tool_search should not appear in MCP tool descriptions") + } +} + +func TestBuildPreviewPrompt_AliasExclusion(t *testing.T) { + r := BuildPreviewPrompt(context.Background(), baseAgent(), PromptFull, "", PreviewDeps{ + ToolLister: &mockToolLister{ + tools: map[string]string{ + "read_file": "Read a file", + "Read": "Alias for read_file", + "exec": "Execute shell", + "Bash": "Alias for exec", + }, + aliases: map[string]string{ + "Read": "read_file", + "Bash": "exec", + }, + }, + }) + if strings.Contains(r.Prompt, "- Read\n") || strings.Contains(r.Prompt, "- Bash\n") { + t.Error("aliases should be excluded from tool list") + } +} + +func TestBuildPreviewPrompt_SkillManageGating(t *testing.T) { + ag := baseAgent() + r := BuildPreviewPrompt(context.Background(), ag, PromptFull, "", PreviewDeps{ + ToolLister: &mockToolLister{ + tools: map[string]string{ + "read_file": "Read a file", + "skill_manage": "Manage skills", + }, + }, + }) + if strings.Contains(r.Prompt, "skill_manage") { + t.Error("skill_manage should be excluded when skill_evolve is off") + } +} + +func TestBuildPreviewPrompt_SkillManageEnabled(t *testing.T) { + ag := baseAgent() + ag.SkillEvolve = true + r := BuildPreviewPrompt(context.Background(), ag, PromptFull, "", PreviewDeps{ + ToolLister: &mockToolLister{ + tools: map[string]string{ + "read_file": "Read a file", + "skill_manage": "Manage skills", + "skill_search": "Search skills", + }, + }, + }) + if !strings.Contains(r.Prompt, "skill_manage") { + t.Error("skill_manage should be present when skill_evolve is on") + } +} + +func TestBuildPreviewPrompt_ToolPolicyDeny(t *testing.T) { + ag := baseAgent() + ag.ToolsConfig = []byte(`{"deny":["exec","web_fetch"]}`) + r := BuildPreviewPrompt(context.Background(), ag, PromptFull, "", PreviewDeps{ + ToolLister: &mockToolLister{ + tools: map[string]string{ + "read_file": "Read a file", + "exec": "Execute shell", + "web_fetch": "Fetch web page", + }, + }, + }) + if strings.Contains(r.Prompt, "- exec\n") { + t.Error("denied tool 'exec' should be excluded") + } + if strings.Contains(r.Prompt, "- web_fetch\n") { + t.Error("denied tool 'web_fetch' should be excluded") + } + if !strings.Contains(r.Prompt, "read_file") { + t.Error("non-denied tool 'read_file' should be present") + } +} + +func TestBuildPreviewPrompt_ToolDefs(t *testing.T) { + r := BuildPreviewPrompt(context.Background(), baseAgent(), PromptFull, "", PreviewDeps{ + ToolLister: &mockToolLister{ + tools: map[string]string{ + "read_file": "Read a file", + "exec": "Execute shell", + }, + aliases: map[string]string{ + "Read": "read_file", + }, + }, + }) + // Should have canonical tools + aliases in tool defs + if len(r.ToolDefs) != 3 { // read_file + exec + Read alias + t.Errorf("expected 3 tool defs (2 canonical + 1 alias), got %d", len(r.ToolDefs)) + } + // Verify alias is included in defs even though excluded from system prompt + found := false + for _, td := range r.ToolDefs { + if td.Function.Name == "Read" { + found = true + break + } + } + if !found { + t.Error("expected alias 'Read' in tool defs") + } +} diff --git a/internal/agent/preview_prompt_test_helpers_test.go b/internal/agent/preview_prompt_test_helpers_test.go new file mode 100644 index 00000000..e5705a32 --- /dev/null +++ b/internal/agent/preview_prompt_test_helpers_test.go @@ -0,0 +1,78 @@ +package agent + +import ( + "context" + "sort" + + "github.com/google/uuid" + + "github.com/nextlevelbuilder/goclaw/internal/store" + "github.com/nextlevelbuilder/goclaw/internal/tools" +) + +// mockToolLister implements the widened ToolLister interface for testing. +type mockToolLister struct { + tools map[string]string // name → description + aliases map[string]string // alias → canonical +} + +func (m *mockToolLister) List() []string { + names := make([]string, 0, len(m.tools)) + for n := range m.tools { + names = append(names, n) + } + sort.Strings(names) + return names +} + +func (m *mockToolLister) Get(name string) (tools.Tool, bool) { + desc, ok := m.tools[name] + if !ok { + return nil, false + } + return &mockTool{name: name, desc: desc}, true +} + +func (m *mockToolLister) Aliases() map[string]string { + if m.aliases == nil { + return nil + } + return m.aliases +} + +// mockTool is a minimal tools.Tool implementation for testing. +type mockTool struct { + name string + desc string +} + +func (t *mockTool) Name() string { return t.name } +func (t *mockTool) Description() string { return t.desc } +func (t *mockTool) Parameters() map[string]any { return nil } +func (t *mockTool) Execute(_ context.Context, _ map[string]any) *tools.Result { return nil } + +// mockSkillsLoader implements the widened SkillsLoader interface. +type mockSkillsLoader struct { + pinned string // pre-built pinned XML + summary string // pre-built full summary + capturedAllow []string // set by BuildSummary for test assertions +} + +func (m *mockSkillsLoader) BuildPinnedSummary(_ context.Context, _ []string) string { + return m.pinned +} + +func (m *mockSkillsLoader) BuildSummary(_ context.Context, allowList []string) string { + m.capturedAllow = allowList + return m.summary +} + +// mockSkillAccessStore returns canned skill access lists. +type mockSkillAccessStore struct { + accessible []store.SkillInfo + err error +} + +func (m *mockSkillAccessStore) ListAccessible(_ context.Context, _ uuid.UUID, _ string) ([]store.SkillInfo, error) { + return m.accessible, m.err +} diff --git a/internal/config/config_channels.go b/internal/config/config_channels.go index 0f31e5f1..037bc790 100644 --- a/internal/config/config_channels.go +++ b/internal/config/config_channels.go @@ -359,6 +359,8 @@ type GatewayConfig struct { BlockReply *bool `json:"block_reply,omitempty"` // deliver intermediate text during tool iterations (default false) ToolStatus *bool `json:"tool_status,omitempty"` // show tool name in streaming preview during tool execution (default true) TaskRecoveryIntervalSec int `json:"task_recovery_interval_sec,omitempty"` // team task recovery ticker interval in seconds (default 300 = 5min) + BackgroundProvider string `json:"background_provider,omitempty"` // LLM provider for background workers (vault enrichment, consolidation) + BackgroundModel string `json:"background_model,omitempty"` // LLM model for background workers } // ToolsConfig controls tool availability, policy, and web search. diff --git a/internal/config/config_system.go b/internal/config/config_system.go index 8027a9ea..c3e576aa 100644 --- a/internal/config/config_system.go +++ b/internal/config/config_system.go @@ -52,6 +52,10 @@ func (c *Config) ApplySystemConfigs(configs map[string]string) { boolean("gateway.tool_status", &c.Gateway.ToolStatus) integer("gateway.task_recovery_interval_sec", &c.Gateway.TaskRecoveryIntervalSec) + // Background workers (vault enrichment, consolidation) + str("background.provider", &c.Gateway.BackgroundProvider) + str("background.model", &c.Gateway.BackgroundModel) + // Tools str("tools.profile", &c.Tools.Profile) integer("tools.rate_limit_per_hour", &c.Tools.RateLimitPerHour) diff --git a/internal/http/agents.go b/internal/http/agents.go index b7bee08d..675332e7 100644 --- a/internal/http/agents.go +++ b/internal/http/agents.go @@ -18,6 +18,7 @@ import ( "github.com/nextlevelbuilder/goclaw/internal/permissions" "github.com/nextlevelbuilder/goclaw/internal/providers" "github.com/nextlevelbuilder/goclaw/internal/store" + "github.com/nextlevelbuilder/goclaw/internal/tools" "github.com/nextlevelbuilder/goclaw/pkg/protocol" ) @@ -32,8 +33,9 @@ type AgentsHandler struct { kgStore store.KnowledgeGraphStore // for import (nil = disabled) episodicStore store.EpisodicStore // for import (nil in SQLite/lite builds) vaultStore store.VaultStore // for vault import (nil = disabled) - toolsReg ToolLister // for system prompt preview tool resolution (nil = fallback) - skillsLoader SkillPinnedBuilder // for system prompt preview pinned skills (nil = skip) + toolsReg ToolPreviewLister // for system prompt preview tool resolution (nil = fallback) + skillsLoader SkillPreviewBuilder // for system prompt preview pinned skills (nil = skip) + skillAccessStore store.SkillAccessStore // for system prompt preview skill filtering (nil = skip) teamStore store.TeamStore // for system prompt preview team context (nil = skip) agentLinkStore store.AgentLinkStore // for system prompt preview delegation targets (nil = skip) defaultWorkspace string // default workspace path template (e.g. "~/.goclaw/workspace") @@ -82,24 +84,30 @@ func (h *AgentsHandler) SetVaultStore(vs store.VaultStore) { h.vaultStore = vs } -// ToolLister is satisfied by tools.Registry for system prompt preview. -type ToolLister interface{ List() []string } +// ToolPreviewLister is satisfied by tools.Registry for system prompt preview. +type ToolPreviewLister interface { + List() []string + Get(name string) (tools.Tool, bool) + Aliases() map[string]string +} -// SkillPinnedBuilder is satisfied by skills.Loader for pinned skills summary. -type SkillPinnedBuilder interface { +// SkillPreviewBuilder is satisfied by skills.Loader for system prompt preview. +type SkillPreviewBuilder interface { BuildPinnedSummary(ctx context.Context, names []string) string + BuildSummary(ctx context.Context, allowList []string) string } // SetPreviewDeps attaches optional dependencies for system prompt preview. -func (h *AgentsHandler) SetPreviewDeps(tl ToolLister, sl SkillPinnedBuilder) { +func (h *AgentsHandler) SetPreviewDeps(tl ToolPreviewLister, sl SkillPreviewBuilder) { h.toolsReg = tl h.skillsLoader = sl } // SetPreviewStores attaches team + agent link stores for system prompt preview. -func (h *AgentsHandler) SetPreviewStores(ts store.TeamStore, als store.AgentLinkStore) { +func (h *AgentsHandler) SetPreviewStores(ts store.TeamStore, als store.AgentLinkStore, sas store.SkillAccessStore) { h.teamStore = ts h.agentLinkStore = als + h.skillAccessStore = sas } // isOwnerUser checks if the given user ID is a system owner. diff --git a/internal/http/agents_import_sections.go b/internal/http/agents_import_sections.go index 91b303fe..9f5d8a34 100644 --- a/internal/http/agents_import_sections.go +++ b/internal/http/agents_import_sections.go @@ -354,9 +354,10 @@ func (h *AgentsHandler) importVault(ctx context.Context, ag *store.AgentData, ar progressFn(ProgressEvent{Phase: "vault_documents", Status: "running", Total: len(arc.vaultDocuments)}) } for _, d := range arc.vaultDocuments { + agentIDStr := ag.ID.String() doc := &store.VaultDocument{ TenantID: tid.String(), - AgentID: ag.ID.String(), + AgentID: &agentIDStr, TeamID: nil, // team_id not portable Scope: d.Scope, CustomScope: d.CustomScope, diff --git a/internal/http/agents_prompt_preview.go b/internal/http/agents_prompt_preview.go index 95c0b4e3..c5a04526 100644 --- a/internal/http/agents_prompt_preview.go +++ b/internal/http/agents_prompt_preview.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/nextlevelbuilder/goclaw/internal/agent" + "github.com/nextlevelbuilder/goclaw/internal/providers" "github.com/nextlevelbuilder/goclaw/internal/tokencount" ) @@ -18,10 +19,11 @@ type promptPreviewSection struct { // promptPreviewResponse is the API response for system prompt preview. type promptPreviewResponse struct { - Mode string `json:"mode"` - Prompt string `json:"prompt"` - TokenCount int `json:"token_count"` - Sections []promptPreviewSection `json:"sections"` + Mode string `json:"mode"` + Prompt string `json:"prompt"` + TokenCount int `json:"token_count"` + Sections []promptPreviewSection `json:"sections"` + Tools []providers.ToolDefinition `json:"tools,omitempty"` } // handleSystemPromptPreview renders the actual system prompt for an agent in a given mode. @@ -49,26 +51,28 @@ func (h *AgentsHandler) handleSystemPromptPreview(w http.ResponseWriter, r *http // Build preview prompt — reuses same BuildSystemPrompt() as LLM pipeline. // Runtime-only fields (channel, peer kind, credentials) are zero-valued; // BuildSystemPrompt nil-checks every field so these sections are simply skipped. - prompt := agent.BuildPreviewPrompt(ctx, ag, mode, r.URL.Query().Get("user_id"), agent.PreviewDeps{ - AgentStore: h.agents, - TeamStore: h.teamStore, - AgentLinks: h.agentLinkStore, - ProviderReg: h.providerReg, - ToolLister: h.toolsReg, - SkillsLoader: h.skillsLoader, - DataDir: h.dataDir, + result := agent.BuildPreviewPrompt(ctx, ag, mode, r.URL.Query().Get("user_id"), agent.PreviewDeps{ + AgentStore: h.agents, + TeamStore: h.teamStore, + AgentLinks: h.agentLinkStore, + ProviderReg: h.providerReg, + ToolLister: h.toolsReg, + SkillsLoader: h.skillsLoader, + SkillAccessStore: h.skillAccessStore, + DataDir: h.dataDir, }) counter := tokencount.NewFallbackCounter() - tokens := counter.Count("claude-3", prompt) - sections := parseSections(prompt) + tokens := counter.Count("claude-3", result.Prompt) + sections := parseSections(result.Prompt) w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(promptPreviewResponse{ Mode: string(mode), - Prompt: prompt, + Prompt: result.Prompt, TokenCount: tokens, Sections: sections, + Tools: result.ToolDefs, }) } diff --git a/internal/http/vault_handlers.go b/internal/http/vault_handlers.go index bcdef4ad..6bc798be 100644 --- a/internal/http/vault_handlers.go +++ b/internal/http/vault_handlers.go @@ -4,7 +4,6 @@ import ( "context" "log/slog" "net/http" - "path/filepath" "strconv" "strings" "sync" @@ -23,17 +22,29 @@ type vaultDocListResponse struct { Total int `json:"total"` } +// AgentLister is the subset of AgentStore needed by VaultHandler (rescan agent_key→UUID mapping). +type AgentLister interface { + List(ctx context.Context, ownerID string) ([]store.AgentData, error) +} + +// TeamLister is the subset of TeamStore needed by VaultHandler (rescan team validation). +type TeamLister interface { + ListTeams(ctx context.Context) ([]store.TeamData, error) +} + // VaultHandler serves Knowledge Vault document and link endpoints. type VaultHandler struct { store store.VaultStore teamAccess store.TeamAccessStore // nil = skip team membership validation (e.g. lite edition) + agents AgentLister // nil = rescan skips agent resolution + teams TeamLister // nil = rescan skips team resolution workspace string eventBus eventbus.DomainEventBus - rescanMu sync.Map // key: agentID → struct{}, per-agent concurrency guard + rescanMu sync.Map // key: tenantID → struct{}, per-tenant concurrency guard } -func NewVaultHandler(s store.VaultStore, ta store.TeamAccessStore, workspace string, bus eventbus.DomainEventBus) *VaultHandler { - return &VaultHandler{store: s, teamAccess: ta, workspace: workspace, eventBus: bus} +func NewVaultHandler(s store.VaultStore, ta store.TeamAccessStore, workspace string, bus eventbus.DomainEventBus, agents AgentLister, teams TeamLister) *VaultHandler { + return &VaultHandler{store: s, teamAccess: ta, agents: agents, teams: teams, workspace: workspace, eventBus: bus} } // validateTeamMembership checks that the requesting user belongs to the given team. @@ -100,7 +111,8 @@ func (h *VaultHandler) RegisterRoutes(mux *http.ServeMux) { mux.HandleFunc("POST /v1/agents/{agentID}/vault/documents", h.auth(h.handleCreateDocument)) mux.HandleFunc("PUT /v1/agents/{agentID}/vault/documents/{docID}", h.auth(h.handleUpdateDocument)) mux.HandleFunc("DELETE /v1/agents/{agentID}/vault/documents/{docID}", h.auth(h.handleDeleteDocument)) - mux.HandleFunc("POST /v1/agents/{agentID}/vault/rescan", h.auth(h.handleRescan)) + mux.HandleFunc("POST /v1/vault/rescan", h.auth(h.handleRescan)) + mux.HandleFunc("POST /v1/vault/search", h.auth(h.handleSearchAll)) mux.HandleFunc("POST /v1/agents/{agentID}/vault/search", h.auth(h.handleSearch)) mux.HandleFunc("GET /v1/agents/{agentID}/vault/documents/{docID}/links", h.auth(h.handleGetLinks)) mux.HandleFunc("POST /v1/agents/{agentID}/vault/links", h.auth(h.handleCreateLink)) @@ -209,7 +221,7 @@ func (h *VaultHandler) handleGetDocument(w http.ResponseWriter, r *http.Request) writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) return } - if doc == nil || doc.AgentID != agentID { + if doc == nil || (doc.AgentID != nil && *doc.AgentID != agentID) { writeJSON(w, http.StatusNotFound, map[string]string{"error": "document not found"}) return } @@ -222,14 +234,24 @@ func (h *VaultHandler) handleGetDocument(w http.ResponseWriter, r *http.Request) writeJSON(w, http.StatusOK, doc) } -// handleSearch runs hybrid FTS+vector search on vault documents. +// handleSearchAll runs tenant-wide search (agent_id optional in body). +func (h *VaultHandler) handleSearchAll(w http.ResponseWriter, r *http.Request) { + h.doSearch(w, r, "") +} + +// handleSearch runs hybrid FTS+vector search scoped to a specific agent. func (h *VaultHandler) handleSearch(w http.ResponseWriter, r *http.Request) { + h.doSearch(w, r, r.PathValue("agentID")) +} + +// doSearch is the shared search implementation for both per-agent and tenant-wide endpoints. +func (h *VaultHandler) doSearch(w http.ResponseWriter, r *http.Request, agentID string) { locale := extractLocale(r) tenantID := store.TenantIDFromContext(r.Context()) - agentID := r.PathValue("agentID") var body struct { Query string `json:"query"` + AgentID string `json:"agent_id"` Scope string `json:"scope"` DocTypes []string `json:"doc_types"` MaxResults int `json:"max_results"` @@ -245,6 +267,10 @@ func (h *VaultHandler) handleSearch(w http.ResponseWriter, r *http.Request) { if body.MaxResults <= 0 { body.MaxResults = 10 } + // Body agent_id only used when path doesn't provide one (tenant-wide endpoint). + if agentID == "" { + agentID = body.AgentID + } searchOpts := store.VaultSearchOptions{ Query: body.Query, @@ -374,7 +400,7 @@ func (h *VaultHandler) handleCreateDocument(w http.ResponseWriter, r *http.Reque doc := &store.VaultDocument{ TenantID: tenantID.String(), - AgentID: agentID, + AgentID: &agentID, Path: body.Path, Title: body.Title, DocType: body.DocType, @@ -412,7 +438,7 @@ func (h *VaultHandler) handleUpdateDocument(w http.ResponseWriter, r *http.Reque docID := r.PathValue("docID") existing, err := h.store.GetDocumentByID(r.Context(), tenantID.String(), docID) - if err != nil || existing == nil || existing.AgentID != agentID { + if err != nil || existing == nil || (existing.AgentID != nil && *existing.AgentID != agentID) { writeJSON(w, http.StatusNotFound, map[string]string{"error": "document not found"}) return } @@ -475,7 +501,7 @@ func (h *VaultHandler) handleDeleteDocument(w http.ResponseWriter, r *http.Reque docID := r.PathValue("docID") existing, err := h.store.GetDocumentByID(r.Context(), tenantID.String(), docID) - if err != nil || existing == nil || existing.AgentID != agentID { + if err != nil || existing == nil || (existing.AgentID != nil && *existing.AgentID != agentID) { writeJSON(w, http.StatusNotFound, map[string]string{"error": "document not found"}) return } @@ -489,7 +515,12 @@ func (h *VaultHandler) handleDeleteDocument(w http.ResponseWriter, r *http.Reque // DeleteDocument without RunContext applies no team_id filter (broad match on tenant+agent+path). // This is safe because we pre-validated team membership above and use server-derived existing.Path. - if err := h.store.DeleteDocument(r.Context(), tenantID.String(), agentID, existing.Path); err != nil { + // Use the doc's actual agent_id (may be empty for team/shared docs). + deleteAgentID := "" + if existing.AgentID != nil { + deleteAgentID = *existing.AgentID + } + if err := h.store.DeleteDocument(r.Context(), tenantID.String(), deleteAgentID, existing.Path); err != nil { slog.Warn("vault.delete failed", "error", err) writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) return @@ -527,7 +558,7 @@ func (h *VaultHandler) handleCreateLink(w http.ResponseWriter, r *http.Request) writeJSON(w, http.StatusNotFound, map[string]string{"error": "one or both documents not found"}) return } - if from.AgentID != agentID { + if from.AgentID != nil && *from.AgentID != agentID { writeJSON(w, http.StatusForbidden, map[string]string{"error": "source document does not belong to this agent"}) return } @@ -564,38 +595,38 @@ func (h *VaultHandler) handleDeleteLink(w http.ResponseWriter, r *http.Request) w.WriteHeader(http.StatusNoContent) } -// handleRescan walks agent workspace and registers missing/changed files in vault. +// handleRescan walks the entire tenant workspace and registers missing/changed files in vault. +// Infers agent/team ownership from directory structure: agents/{key}/, teams/{uuid}/, or root shared. func (h *VaultHandler) handleRescan(w http.ResponseWriter, r *http.Request) { - agentID := r.PathValue("agentID") - if _, err := uuid.Parse(agentID); err != nil { - writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid agent_id"}) - return - } + tenantID := store.TenantIDFromContext(r.Context()).String() - // Per-agent concurrency guard: only one rescan at a time. - if _, loaded := h.rescanMu.LoadOrStore(agentID, struct{}{}); loaded { + // Per-tenant concurrency guard. + if _, loaded := h.rescanMu.LoadOrStore(tenantID, struct{}{}); loaded { writeJSON(w, http.StatusConflict, map[string]string{"error": "rescan already in progress"}) return } - defer h.rescanMu.Delete(agentID) + defer h.rescanMu.Delete(tenantID) - // Resolve workspace path for this agent. - wsPath := h.resolveAgentWorkspace(r.Context(), agentID) + wsPath := h.resolveTenantWorkspace(r.Context()) if wsPath == "" { writeJSON(w, http.StatusBadRequest, map[string]string{"error": "workspace not available"}) return } + // Build agent_key→UUID map and team UUID set for path inference. + agentMap, teamSet := h.buildRescanMaps(r.Context()) + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Minute) defer cancel() result, err := vault.RescanWorkspace(ctx, vault.RescanParams{ - TenantID: store.TenantIDFromContext(r.Context()).String(), - AgentID: agentID, + TenantID: tenantID, Workspace: wsPath, + AgentMap: agentMap, + TeamSet: teamSet, }, h.store, h.eventBus) if err != nil { - slog.Warn("vault.rescan failed", "agent", agentID, "error", err) + slog.Warn("vault.rescan failed", "tenant", tenantID, "error", err) writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) return } @@ -603,16 +634,38 @@ func (h *VaultHandler) handleRescan(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, result) } -// resolveAgentWorkspace returns the filesystem path to an agent's workspace. -// Scopes by tenant to prevent cross-tenant file access. -func (h *VaultHandler) resolveAgentWorkspace(ctx context.Context, agentID string) string { +// resolveTenantWorkspace returns the tenant-scoped workspace root. +func (h *VaultHandler) resolveTenantWorkspace(ctx context.Context) string { if h.workspace == "" { return "" } tenantID := store.TenantIDFromContext(ctx) slug := store.TenantSlugFromContext(ctx) - ws := config.TenantWorkspace(h.workspace, tenantID, slug) - return filepath.Join(ws, agentID) + return config.TenantWorkspace(h.workspace, tenantID, slug) +} + +// buildRescanMaps pre-loads agent_key→UUID and team UUID sets for the current tenant. +func (h *VaultHandler) buildRescanMaps(ctx context.Context) (map[string]string, map[string]bool) { + agentMap := make(map[string]string) + teamSet := make(map[string]bool) + + if h.agents != nil { + agents, err := h.agents.List(ctx, "") + if err == nil { + for _, a := range agents { + agentMap[a.AgentKey] = a.ID.String() + } + } + } + if h.teams != nil { + teams, err := h.teams.ListTeams(ctx) + if err == nil { + for _, t := range teams { + teamSet[t.ID.String()] = true + } + } + } + return agentMap, teamSet } var allowedDocTypes = map[string]bool{"context": true, "memory": true, "note": true, "skill": true, "episodic": true, "media": true} diff --git a/internal/store/pg/vault_documents.go b/internal/store/pg/vault_documents.go index 7784766c..86457608 100644 --- a/internal/store/pg/vault_documents.go +++ b/internal/store/pg/vault_documents.go @@ -54,10 +54,19 @@ func (s *PGVaultStore) SetEmbeddingProvider(provider store.EmbeddingProvider) { func (s *PGVaultStore) Close() error { return nil } +// optAgentUUID converts a nullable *string agent_id to *uuid.UUID for SQL. +func optAgentUUID(agentID *string) *uuid.UUID { + if agentID == nil || *agentID == "" { + return nil + } + u := mustParseUUID(*agentID) + return &u +} + // UpsertDocument inserts or updates a vault document. func (s *PGVaultStore) UpsertDocument(ctx context.Context, doc *store.VaultDocument) error { tid := mustParseUUID(doc.TenantID) - aid := mustParseUUID(doc.AgentID) + aid := optAgentUUID(doc.AgentID) now := time.Now().UTC() meta, err := json.Marshal(doc.Metadata) @@ -91,7 +100,7 @@ func (s *PGVaultStore) UpsertDocument(ctx context.Context, doc *store.VaultDocum INSERT INTO vault_documents (id, tenant_id, agent_id, team_id, scope, custom_scope, path, title, doc_type, content_hash, summary, embedding, metadata, created_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $14) - ON CONFLICT (agent_id, COALESCE(team_id, '00000000-0000-0000-0000-000000000000'), scope, path) DO UPDATE SET + ON CONFLICT (tenant_id, COALESCE(agent_id, '00000000-0000-0000-0000-000000000000'::uuid), COALESCE(team_id, '00000000-0000-0000-0000-000000000000'::uuid), scope, path) DO UPDATE SET title = EXCLUDED.title, doc_type = EXCLUDED.doc_type, content_hash = EXCLUDED.content_hash, @@ -112,15 +121,21 @@ func (s *PGVaultStore) UpsertDocument(ctx context.Context, doc *store.VaultDocum } // GetDocument retrieves a vault document by tenant, agent, and path. +// Empty agentID means no agent filter (match any agent). // Team scoping via RunContext: present+TeamID → filter; present+empty → personal; nil → any match. func (s *PGVaultStore) GetDocument(ctx context.Context, tenantID, agentID, path string) (*store.VaultDocument, error) { tid := mustParseUUID(tenantID) - aid := mustParseUUID(agentID) q := `SELECT id, tenant_id, agent_id, team_id, scope, custom_scope, path, title, doc_type, content_hash, summary, metadata, created_at, updated_at - FROM vault_documents WHERE tenant_id = $1 AND agent_id = $2 AND path = $3` - args := []any{tid, aid, path} - p := 4 + FROM vault_documents WHERE tenant_id = $1 AND path = $2` + args := []any{tid, path} + p := 3 + + if agentID != "" { + q += fmt.Sprintf(" AND agent_id = $%d", p) + args = append(args, mustParseUUID(agentID)) + p++ + } if rc := store.RunContextFromCtx(ctx); rc != nil { if rc.TeamID != "" { @@ -162,14 +177,20 @@ func (s *PGVaultStore) GetDocumentByID(ctx context.Context, tenantID, id string) } // DeleteDocument removes a vault document by tenant, agent, and path. +// Empty agentID means no agent filter. // Team scoping via RunContext (same rules as GetDocument). func (s *PGVaultStore) DeleteDocument(ctx context.Context, tenantID, agentID, path string) error { tid := mustParseUUID(tenantID) - aid := mustParseUUID(agentID) - q := `DELETE FROM vault_documents WHERE tenant_id = $1 AND agent_id = $2 AND path = $3` - args := []any{tid, aid, path} - p := 4 + q := `DELETE FROM vault_documents WHERE tenant_id = $1 AND path = $2` + args := []any{tid, path} + p := 3 + + if agentID != "" { + q += fmt.Sprintf(" AND agent_id = $%d", p) + args = append(args, mustParseUUID(agentID)) + p++ + } if rc := store.RunContextFromCtx(ctx); rc != nil { if rc.TeamID != "" { @@ -287,7 +308,7 @@ func (s *PGVaultStore) UpdateHash(ctx context.Context, tenantID, id, newHash str // Search performs hybrid FTS + vector search on vault_documents. func (s *PGVaultStore) Search(ctx context.Context, opts store.VaultSearchOptions) ([]store.VaultSearchResult, error) { tid := mustParseUUID(opts.TenantID) - aid := mustParseUUID(opts.AgentID) + aid := optAgentUUID(&opts.AgentID) // empty string → nil → no agent filter // Build team filter for search sub-queries. tf := buildSearchTeamFilter(opts.TeamID, opts.TeamIDs) @@ -381,13 +402,19 @@ func (tf searchTeamFilter) append(q string, args []any, p int) (string, []any, i return q, args, p } -func (s *PGVaultStore) ftsSearch(ctx context.Context, query string, tenantID, agentID uuid.UUID, tf searchTeamFilter, scope string, docTypes []string, limit int) ([]store.VaultSearchResult, error) { +func (s *PGVaultStore) ftsSearch(ctx context.Context, query string, tenantID uuid.UUID, agentID *uuid.UUID, tf searchTeamFilter, scope string, docTypes []string, limit int) ([]store.VaultSearchResult, error) { q := `SELECT id, tenant_id, agent_id, team_id, scope, custom_scope, path, title, doc_type, content_hash, summary, metadata, created_at, updated_at, ts_rank(tsv, plainto_tsquery('simple', $1)) AS score FROM vault_documents - WHERE tenant_id = $2 AND agent_id = $3 AND tsv @@ plainto_tsquery('simple', $1)` - args := []any{query, tenantID, agentID} - p := 4 + WHERE tenant_id = $2 AND tsv @@ plainto_tsquery('simple', $1)` + args := []any{query, tenantID} + p := 3 + + if agentID != nil { + q += fmt.Sprintf(" AND agent_id = $%d", p) + args = append(args, *agentID) + p++ + } q, args, p = tf.append(q, args, p) @@ -412,14 +439,20 @@ func (s *PGVaultStore) ftsSearch(ctx context.Context, query string, tenantID, ag return vaultSearchRowsToResults(scanned, "vault"), nil } -func (s *PGVaultStore) vectorSearch(ctx context.Context, embedding []float32, tenantID, agentID uuid.UUID, tf searchTeamFilter, scope string, docTypes []string, limit int) ([]store.VaultSearchResult, error) { +func (s *PGVaultStore) vectorSearch(ctx context.Context, embedding []float32, tenantID uuid.UUID, agentID *uuid.UUID, tf searchTeamFilter, scope string, docTypes []string, limit int) ([]store.VaultSearchResult, error) { vecStr := vectorToString(embedding) q := `SELECT id, tenant_id, agent_id, team_id, scope, custom_scope, path, title, doc_type, content_hash, summary, metadata, created_at, updated_at, 1 - (embedding <=> $1) AS score FROM vault_documents - WHERE tenant_id = $2 AND agent_id = $3 AND embedding IS NOT NULL` - args := []any{vecStr, tenantID, agentID} - p := 4 + WHERE tenant_id = $2 AND embedding IS NOT NULL` + args := []any{vecStr, tenantID} + p := 3 + + if agentID != nil { + q += fmt.Sprintf(" AND agent_id = $%d", p) + args = append(args, *agentID) + p++ + } q, args, p = tf.append(q, args, p) diff --git a/internal/store/pg/vault_documents_enrichment.go b/internal/store/pg/vault_documents_enrichment.go index 9d448583..fbbbf1b0 100644 --- a/internal/store/pg/vault_documents_enrichment.go +++ b/internal/store/pg/vault_documents_enrichment.go @@ -44,9 +44,10 @@ func (s *PGVaultStore) UpdateSummaryAndReembed(ctx context.Context, tenantID, do // FindSimilarDocs finds documents with similar embeddings to the given docID. // Returns top-N neighbors excluding the source doc itself. +// Empty agentID means no agent filter. func (s *PGVaultStore) FindSimilarDocs(ctx context.Context, tenantID, agentID, docID string, limit int) ([]store.VaultSearchResult, error) { tid := mustParseUUID(tenantID) - aid := mustParseUUID(agentID) + aid := optAgentUUID(&agentID) did := mustParseUUID(docID) // Fetch source embedding. @@ -63,12 +64,20 @@ func (s *PGVaultStore) FindSimilarDocs(ctx context.Context, tenantID, agentID, d content_hash, summary, metadata, created_at, updated_at, 1 - (embedding <=> $1::vector) AS score FROM vault_documents - WHERE tenant_id = $2 AND agent_id = $3 AND id != $4 AND embedding IS NOT NULL - ORDER BY embedding <=> $1::vector - LIMIT $5` + WHERE tenant_id = $2 AND id != $3 AND embedding IS NOT NULL` + args := []any{*embStr, tid, did} + p := 4 + + if aid != nil { + q += fmt.Sprintf(" AND agent_id = $%d", p) + args = append(args, *aid) + p++ + } + q += fmt.Sprintf(" ORDER BY embedding <=> $1::vector LIMIT $%d", p) + args = append(args, limit) var scanned []vaultSearchRow - if err := pkgSqlxDB.SelectContext(ctx, &scanned, q, *embStr, tid, aid, did, limit); err != nil { + if err := pkgSqlxDB.SelectContext(ctx, &scanned, q, args...); err != nil { return nil, fmt.Errorf("vault.find_similar: %w", err) } return vaultSearchRowsToResults(scanned, "vault"), nil diff --git a/internal/store/pg/vault_scan_rows.go b/internal/store/pg/vault_scan_rows.go index 1748729c..96ea271c 100644 --- a/internal/store/pg/vault_scan_rows.go +++ b/internal/store/pg/vault_scan_rows.go @@ -14,7 +14,7 @@ import ( type vaultDocRow struct { ID uuid.UUID `db:"id"` TenantID uuid.UUID `db:"tenant_id"` - AgentID uuid.UUID `db:"agent_id"` + AgentID *uuid.UUID `db:"agent_id"` TeamID *uuid.UUID `db:"team_id"` Scope string `db:"scope"` CustomScope *string `db:"custom_scope"` @@ -33,7 +33,6 @@ func (r *vaultDocRow) toVaultDocument() store.VaultDocument { doc := store.VaultDocument{ ID: r.ID.String(), TenantID: r.TenantID.String(), - AgentID: r.AgentID.String(), Scope: r.Scope, CustomScope: r.CustomScope, Path: r.Path, @@ -44,6 +43,10 @@ func (r *vaultDocRow) toVaultDocument() store.VaultDocument { CreatedAt: r.CreatedAt, UpdatedAt: r.UpdatedAt, } + if r.AgentID != nil { + s := r.AgentID.String() + doc.AgentID = &s + } if r.TeamID != nil { s := r.TeamID.String() doc.TeamID = &s diff --git a/internal/store/sqlitestore/schema.go b/internal/store/sqlitestore/schema.go index 5f664ecd..5ae63f90 100644 --- a/internal/store/sqlitestore/schema.go +++ b/internal/store/sqlitestore/schema.go @@ -15,7 +15,7 @@ var schemaSQL string // SchemaVersion is the current SQLite schema version. // Bump this when adding new migration steps below. -const SchemaVersion = 13 +const SchemaVersion = 14 // migrations maps version → SQL to apply when upgrading FROM that version. // schema.sql always represents the LATEST full schema (for fresh DBs). @@ -374,6 +374,36 @@ ALTER TABLE episodic_summaries ADD COLUMN recall_score REAL NOT NULL DEFAULT 0; ALTER TABLE episodic_summaries ADD COLUMN last_recalled_at TEXT; CREATE INDEX IF NOT EXISTS idx_episodic_recall_unpromoted ON episodic_summaries(agent_id, user_id, recall_score DESC) WHERE promoted_at IS NULL;`, + + // Version 13 → 14: vault_documents agent_id nullable + unique index with tenant_id. + // SQLite requires table recreation to drop NOT NULL. Preserve all data. + 13: `CREATE TABLE vault_documents_new ( + id TEXT NOT NULL PRIMARY KEY, + tenant_id TEXT NOT NULL REFERENCES tenants(id) ON DELETE CASCADE, + agent_id TEXT REFERENCES agents(id) ON DELETE SET NULL, + team_id TEXT REFERENCES agent_teams(id) ON DELETE SET NULL, + scope TEXT NOT NULL DEFAULT 'personal', + custom_scope TEXT, + path TEXT NOT NULL, + title TEXT NOT NULL DEFAULT '', + doc_type TEXT NOT NULL DEFAULT 'note', + content_hash TEXT NOT NULL DEFAULT '', + summary TEXT NOT NULL DEFAULT '', + metadata TEXT DEFAULT '{}', + created_at TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) +); +INSERT INTO vault_documents_new SELECT * FROM vault_documents; +DROP TABLE vault_documents; +ALTER TABLE vault_documents_new RENAME TO vault_documents; +DROP INDEX IF EXISTS idx_vault_docs_unique_path; +CREATE UNIQUE INDEX idx_vault_docs_unique_path + ON vault_documents(tenant_id, COALESCE(agent_id, ''), COALESCE(team_id, ''), scope, path); +CREATE INDEX IF NOT EXISTS idx_vault_docs_tenant ON vault_documents(tenant_id); +CREATE INDEX IF NOT EXISTS idx_vault_docs_agent_scope ON vault_documents(agent_id, scope); +CREATE INDEX IF NOT EXISTS idx_vault_docs_type ON vault_documents(agent_id, doc_type); +CREATE INDEX IF NOT EXISTS idx_vault_docs_hash ON vault_documents(content_hash); +CREATE INDEX IF NOT EXISTS idx_vault_docs_team ON vault_documents(team_id);`, } // EnsureSchema creates tables if they don't exist and applies incremental migrations. diff --git a/internal/store/sqlitestore/schema.sql b/internal/store/sqlitestore/schema.sql index c6de6b59..f71e2bae 100644 --- a/internal/store/sqlitestore/schema.sql +++ b/internal/store/sqlitestore/schema.sql @@ -1516,7 +1516,7 @@ CREATE INDEX IF NOT EXISTS idx_scuc_binary ON secure_cli_user_credentials(binary CREATE TABLE IF NOT EXISTS vault_documents ( id TEXT NOT NULL PRIMARY KEY, tenant_id TEXT NOT NULL REFERENCES tenants(id) ON DELETE CASCADE, - agent_id TEXT NOT NULL REFERENCES agents(id) ON DELETE CASCADE, + agent_id TEXT REFERENCES agents(id) ON DELETE SET NULL, team_id TEXT REFERENCES agent_teams(id) ON DELETE SET NULL, scope TEXT NOT NULL DEFAULT 'personal', custom_scope TEXT, @@ -1531,7 +1531,7 @@ CREATE TABLE IF NOT EXISTS vault_documents ( ); -- SQLite prohibits expressions in inline UNIQUE constraints; use a unique index instead. CREATE UNIQUE INDEX IF NOT EXISTS idx_vault_docs_unique_path - ON vault_documents(agent_id, COALESCE(team_id, ''), scope, path); + ON vault_documents(tenant_id, COALESCE(agent_id, ''), COALESCE(team_id, ''), scope, path); CREATE INDEX IF NOT EXISTS idx_vault_docs_tenant ON vault_documents(tenant_id); CREATE INDEX IF NOT EXISTS idx_vault_docs_agent_scope ON vault_documents(agent_id, scope); CREATE INDEX IF NOT EXISTS idx_vault_docs_type ON vault_documents(agent_id, doc_type); diff --git a/internal/store/sqlitestore/vault_documents.go b/internal/store/sqlitestore/vault_documents.go index 5b8a5e7a..86ddc45a 100644 --- a/internal/store/sqlitestore/vault_documents.go +++ b/internal/store/sqlitestore/vault_documents.go @@ -58,11 +58,16 @@ func (s *SQLiteVaultStore) UpsertDocument(ctx context.Context, doc *store.VaultD meta = []byte("{}") } + // Convert nullable *string AgentID to nil for SQL. + var agentIDVal any + if doc.AgentID != nil && *doc.AgentID != "" { + agentIDVal = *doc.AgentID + } err = s.db.QueryRowContext(ctx, ` INSERT INTO vault_documents (id, tenant_id, agent_id, team_id, scope, custom_scope, path, title, doc_type, content_hash, summary, metadata, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT (agent_id, COALESCE(team_id, '00000000-0000-0000-0000-000000000000'), scope, path) DO UPDATE SET + ON CONFLICT (tenant_id, COALESCE(agent_id,''), COALESCE(team_id,''), scope, path) DO UPDATE SET title = excluded.title, doc_type = excluded.doc_type, content_hash = excluded.content_hash, @@ -71,7 +76,7 @@ func (s *SQLiteVaultStore) UpsertDocument(ctx context.Context, doc *store.VaultD tenant_id = excluded.tenant_id, updated_at = excluded.updated_at RETURNING id`, - id, doc.TenantID, doc.AgentID, doc.TeamID, doc.Scope, doc.CustomScope, + id, doc.TenantID, agentIDVal, doc.TeamID, doc.Scope, doc.CustomScope, doc.Path, doc.Title, doc.DocType, doc.ContentHash, doc.Summary, string(meta), now, now, ).Scan(&doc.ID) if err != nil { @@ -81,11 +86,17 @@ func (s *SQLiteVaultStore) UpsertDocument(ctx context.Context, doc *store.VaultD } // GetDocument retrieves a vault document by tenant, agent, and path. +// Empty agentID means no agent filter. // Team scoping via RunContext: present+TeamID → filter; present+empty → personal; nil → any match. func (s *SQLiteVaultStore) GetDocument(ctx context.Context, tenantID, agentID, path string) (*store.VaultDocument, error) { q := `SELECT id, tenant_id, agent_id, team_id, scope, custom_scope, path, title, doc_type, content_hash, summary, metadata, created_at, updated_at - FROM vault_documents WHERE tenant_id = ? AND agent_id = ? AND path = ?` - args := []any{tenantID, agentID, path} + FROM vault_documents WHERE tenant_id = ? AND path = ?` + args := []any{tenantID, path} + + if agentID != "" { + q += " AND agent_id = ?" + args = append(args, agentID) + } if rc := store.RunContextFromCtx(ctx); rc != nil { if rc.TeamID != "" { @@ -109,10 +120,16 @@ func (s *SQLiteVaultStore) GetDocumentByID(ctx context.Context, tenantID, id str } // DeleteDocument removes a vault document (FK cascades delete vault_links). +// Empty agentID means no agent filter. // Team scoping via RunContext (same rules as GetDocument). func (s *SQLiteVaultStore) DeleteDocument(ctx context.Context, tenantID, agentID, path string) error { - q := `DELETE FROM vault_documents WHERE tenant_id = ? AND agent_id = ? AND path = ?` - args := []any{tenantID, agentID, path} + q := `DELETE FROM vault_documents WHERE tenant_id = ? AND path = ?` + args := []any{tenantID, path} + + if agentID != "" { + q += " AND agent_id = ?" + args = append(args, agentID) + } if rc := store.RunContextFromCtx(ctx); rc != nil { if rc.TeamID != "" { @@ -249,9 +266,14 @@ func (s *SQLiteVaultStore) Search(ctx context.Context, opts store.VaultSearchOpt q := `SELECT id, tenant_id, agent_id, team_id, scope, custom_scope, path, title, doc_type, content_hash, summary, metadata, created_at, updated_at FROM vault_documents - WHERE tenant_id = ? AND agent_id = ? + WHERE tenant_id = ? AND (title LIKE ? ESCAPE '\' OR path LIKE ? ESCAPE '\')` - args := []any{opts.TenantID, opts.AgentID, pattern, pattern} + args := []any{opts.TenantID, pattern, pattern} + + if opts.AgentID != "" { + q += " AND agent_id = ?" + args = append(args, opts.AgentID) + } q, args = sqliteAppendTeamFilter(q, args, opts.TeamID, opts.TeamIDs) @@ -304,12 +326,14 @@ func (s *SQLiteVaultStore) Search(ctx context.Context, opts store.VaultSearchOpt func scanVaultDoc(row *sql.Row) (*store.VaultDocument, error) { var doc store.VaultDocument var meta []byte + var agentID *string ca, ua := &sqliteTime{}, &sqliteTime{} - err := row.Scan(&doc.ID, &doc.TenantID, &doc.AgentID, &doc.TeamID, &doc.Scope, &doc.CustomScope, + err := row.Scan(&doc.ID, &doc.TenantID, &agentID, &doc.TeamID, &doc.Scope, &doc.CustomScope, &doc.Path, &doc.Title, &doc.DocType, &doc.ContentHash, &doc.Summary, &meta, ca, ua) if err != nil { return nil, err } + doc.AgentID = agentID doc.CreatedAt = ca.Time doc.UpdatedAt = ua.Time if len(meta) > 2 { @@ -321,12 +345,14 @@ func scanVaultDoc(row *sql.Row) (*store.VaultDocument, error) { func scanVaultDocRow(rows *sql.Rows) (*store.VaultDocument, error) { var doc store.VaultDocument var meta []byte + var agentID *string ca, ua := &sqliteTime{}, &sqliteTime{} - err := rows.Scan(&doc.ID, &doc.TenantID, &doc.AgentID, &doc.TeamID, &doc.Scope, &doc.CustomScope, + err := rows.Scan(&doc.ID, &doc.TenantID, &agentID, &doc.TeamID, &doc.Scope, &doc.CustomScope, &doc.Path, &doc.Title, &doc.DocType, &doc.ContentHash, &doc.Summary, &meta, ca, ua) if err != nil { return nil, err } + doc.AgentID = agentID doc.CreatedAt = ca.Time doc.UpdatedAt = ua.Time if len(meta) > 2 { diff --git a/internal/store/vault_store.go b/internal/store/vault_store.go index 5ac3a0a0..cd9d8e33 100644 --- a/internal/store/vault_store.go +++ b/internal/store/vault_store.go @@ -9,7 +9,7 @@ import ( type VaultDocument struct { ID string `json:"id" db:"id"` TenantID string `json:"tenant_id" db:"tenant_id"` - AgentID string `json:"agent_id" db:"agent_id"` + AgentID *string `json:"agent_id,omitempty" db:"agent_id"` TeamID *string `json:"team_id,omitempty" db:"team_id"` Scope string `json:"scope" db:"scope"` // personal, team, shared CustomScope *string `json:"custom_scope,omitempty" db:"custom_scope"` diff --git a/internal/tools/vault_interceptor.go b/internal/tools/vault_interceptor.go index 246d8b02..e9d6597f 100644 --- a/internal/tools/vault_interceptor.go +++ b/internal/tools/vault_interceptor.go @@ -25,14 +25,15 @@ func NewVaultInterceptor(vs store.VaultStore, workspace string, bus eventbus.Dom return &VaultInterceptor{vaultStore: vs, workspace: workspace, eventBus: bus} } -// inferScopeFromContext returns scope and team_id based on RunContext. -// TeamID present → scope="team", teamID=&rc.TeamID. Absent → "personal", nil. -func inferScopeFromContext(ctx context.Context) (scope string, teamID *string) { +// inferScopeFromContext returns scope, team_id, and whether agent_id should be set. +// TeamID present → scope="team", teamID=&rc.TeamID, agentOwned=false. +// Absent → "personal", nil, agentOwned=true. +func inferScopeFromContext(ctx context.Context) (scope string, teamID *string, agentOwned bool) { rc := store.RunContextFromCtx(ctx) if rc != nil && rc.TeamID != "" { - return "team", &rc.TeamID + return "team", &rc.TeamID, false } - return "personal", nil + return "personal", nil, true } // AfterWrite registers or updates a vault document after a file write. @@ -58,11 +59,19 @@ func (v *VaultInterceptor) AfterWrite(ctx context.Context, resolvedPath, content hash := vault.ContentHash([]byte(content)) title := vault.InferTitle(relPath) docType := vault.InferDocType(relPath) - scope, teamID := inferScopeFromContext(ctx) + scope, teamID, agentOwned := inferScopeFromContext(ctx) + + // Team-scoped files belong to the team, not the creating agent. + var agentIDPtr *string + eventAgentID := "" + if agentOwned { + agentIDPtr = &agentID + eventAgentID = agentID + } doc := &store.VaultDocument{ TenantID: tenantID, - AgentID: agentID, + AgentID: agentIDPtr, TeamID: teamID, Scope: scope, Path: relPath, @@ -80,14 +89,14 @@ func (v *VaultInterceptor) AfterWrite(ctx context.Context, resolvedPath, content v.eventBus.Publish(eventbus.DomainEvent{ ID: uuid.Must(uuid.NewV7()).String(), Type: eventbus.EventVaultDocUpserted, - SourceID: doc.ID + ":" + hash, // unique per content version, avoids bus-level dedup suppression + SourceID: doc.ID + ":" + hash, TenantID: tenantID, - AgentID: agentID, + AgentID: eventAgentID, Timestamp: time.Now(), Payload: eventbus.VaultDocUpsertedPayload{ DocID: doc.ID, TenantID: tenantID, - AgentID: agentID, + AgentID: eventAgentID, Path: relPath, ContentHash: hash, Workspace: v.workspace, @@ -124,11 +133,18 @@ func (v *VaultInterceptor) AfterWriteMedia(ctx context.Context, resolvedPath, su } title := vault.InferTitle(relPath) - scope, teamID := inferScopeFromContext(ctx) + scope, teamID, agentOwned := inferScopeFromContext(ctx) + + var agentIDPtr *string + eventAgentID := "" + if agentOwned { + agentIDPtr = &agentID + eventAgentID = agentID + } doc := &store.VaultDocument{ TenantID: tenantID, - AgentID: agentID, + AgentID: agentIDPtr, TeamID: teamID, Scope: scope, Path: relPath, @@ -150,12 +166,12 @@ func (v *VaultInterceptor) AfterWriteMedia(ctx context.Context, resolvedPath, su Type: eventbus.EventVaultDocUpserted, SourceID: doc.ID + ":" + hash, TenantID: tenantID, - AgentID: agentID, + AgentID: eventAgentID, Timestamp: time.Now(), Payload: eventbus.VaultDocUpsertedPayload{ DocID: doc.ID, TenantID: tenantID, - AgentID: agentID, + AgentID: eventAgentID, Path: relPath, ContentHash: hash, Workspace: v.workspace, @@ -183,7 +199,11 @@ func (v *VaultInterceptor) BeforeRead(ctx context.Context, resolvedPath string) return } + // Try agent-scoped first, then tenant-wide (team/shared docs have no agent_id). doc, err := v.vaultStore.GetDocument(ctx, tenantID, agentID, relPath) + if err != nil { + doc, err = v.vaultStore.GetDocument(ctx, tenantID, "", relPath) + } if err != nil { return // not registered yet — skip } diff --git a/internal/upgrade/version.go b/internal/upgrade/version.go index beadd52e..c20a3461 100644 --- a/internal/upgrade/version.go +++ b/internal/upgrade/version.go @@ -2,4 +2,4 @@ package upgrade // RequiredSchemaVersion is the schema migration version this binary requires. // Bump this whenever adding a new SQL migration file. -const RequiredSchemaVersion uint = 45 +const RequiredSchemaVersion uint = 46 diff --git a/internal/vault/enrich_worker.go b/internal/vault/enrich_worker.go index 25c567cd..2b7a2b91 100644 --- a/internal/vault/enrich_worker.go +++ b/internal/vault/enrich_worker.go @@ -74,7 +74,13 @@ func (w *enrichWorker) Handle(ctx context.Context, event eventbus.DomainEvent) e } w.dedupMu.Unlock() - key := payload.TenantID + ":" + payload.AgentID + // Batch key: tenant + agent for agent-scoped docs. For team/shared docs + // (empty AgentID), use tenant + docID to avoid collapsing all into one queue. + batchScope := payload.AgentID + if batchScope == "" { + batchScope = payload.DocID + } + key := payload.TenantID + ":" + batchScope if !w.queue.Enqueue(key, payload) { return nil // another goroutine already processing this agent's queue } @@ -145,7 +151,6 @@ func (w *enrichWorker) processBatch(ctx context.Context, key string) { } // Phase 3 — Classify links (replaces autoLink). - // All items share same tenantID:agentID (batch queue key guarantees this). if len(embedded) > 0 { first := embedded[0].payload w.classifyLinks(ctx, first.TenantID, first.AgentID, embedded) diff --git a/internal/vault/rescan.go b/internal/vault/rescan.go index 713cc481..06d72154 100644 --- a/internal/vault/rescan.go +++ b/internal/vault/rescan.go @@ -12,11 +12,13 @@ import ( "github.com/nextlevelbuilder/goclaw/internal/store" ) -// RescanParams holds input for workspace rescan. +// RescanParams holds input for tenant-wide workspace rescan. +// AgentMap and TeamSet are pre-loaded by the caller to avoid per-file DB lookups. type RescanParams struct { TenantID string - AgentID string - Workspace string // absolute path to agent's workspace root + Workspace string // absolute path to tenant's workspace root + AgentMap map[string]string // agent_key → agent UUID + TeamSet map[string]bool // team UUID → exists (for validation) } // RescanResult holds the outcome of a workspace rescan. @@ -30,9 +32,10 @@ type RescanResult struct { Truncated bool `json:"truncated"` } -// RescanWorkspace walks the agent workspace and registers missing or changed -// files in vault_documents. Publishes EventVaultDocUpserted for each new or -// updated file so the enrichment worker can process them asynchronously. +// RescanWorkspace walks the tenant workspace and registers missing or changed +// files in vault_documents. Ownership (agent/team/scope) is inferred from path. +// Publishes EventVaultDocUpserted for each new or updated file so the +// enrichment worker can process them asynchronously. func RescanWorkspace(ctx context.Context, params RescanParams, vs store.VaultStore, bus eventbus.DomainEventBus) (*RescanResult, error) { entries, walkStats, err := SafeWalkWorkspace(ctx, params.Workspace, DefaultWalkOptions()) if err != nil { @@ -46,28 +49,40 @@ func RescanWorkspace(ctx context.Context, params RescanParams, vs store.VaultSto } for _, entry := range entries { + agentID, teamID, scope, strippedPath := inferOwnerFromPath(entry.RelPath, params.AgentMap, params.TeamSet) + if scope == "" { + // Unknown agent key or invalid team UUID — skip. + result.Skipped++ + continue + } + hash, hashErr := ContentHashFile(entry.AbsPath) if hashErr != nil { result.Errors++ continue } + // Resolve the agent ID string for store lookup (empty string = no agent filter). + agentIDStr := "" + if agentID != nil { + agentIDStr = *agentID + } + // Check if document already exists with same hash. - existing, _ := vs.GetDocument(ctx, params.TenantID, params.AgentID, entry.RelPath) + existing, _ := vs.GetDocument(ctx, params.TenantID, agentIDStr, strippedPath) if existing != nil && existing.ContentHash == hash { result.Unchanged++ continue } - scope, teamID := inferScopeFromPath(entry.RelPath) doc := &store.VaultDocument{ TenantID: params.TenantID, - AgentID: params.AgentID, + AgentID: agentID, TeamID: teamID, Scope: scope, - Path: entry.RelPath, - Title: InferTitle(entry.RelPath), - DocType: InferDocType(entry.RelPath), + Path: strippedPath, + Title: InferTitle(strippedPath), + DocType: InferDocType(strippedPath), ContentHash: hash, } @@ -83,20 +98,20 @@ func RescanWorkspace(ctx context.Context, params RescanParams, vs store.VaultSto result.New++ } - // Publish enrichment event. + // Publish enrichment event. AgentID in payload stays string for serialization. if bus != nil { bus.Publish(eventbus.DomainEvent{ ID: uuid.Must(uuid.NewV7()).String(), Type: eventbus.EventVaultDocUpserted, SourceID: doc.ID + ":" + hash, TenantID: params.TenantID, - AgentID: params.AgentID, + AgentID: agentIDStr, Timestamp: time.Now(), Payload: eventbus.VaultDocUpsertedPayload{ DocID: doc.ID, TenantID: params.TenantID, - AgentID: params.AgentID, - Path: entry.RelPath, + AgentID: agentIDStr, + Path: strippedPath, ContentHash: hash, Workspace: params.Workspace, }, @@ -104,26 +119,57 @@ func RescanWorkspace(ctx context.Context, params RescanParams, vs store.VaultSto } } - slog.Info("vault.rescan", "agent", params.AgentID, + slog.Info("vault.rescan", + "tenant", params.TenantID, "scanned", result.Scanned, "new", result.New, "updated", result.Updated, "unchanged", result.Unchanged, - "errors", result.Errors, "truncated", result.Truncated) + "skipped", result.Skipped, "errors", result.Errors, + "truncated", result.Truncated) return result, nil } -// inferScopeFromPath detects scope and team from workspace-relative path. -// Paths starting with "teams/{id}/" are team-scoped; everything else is personal. -func inferScopeFromPath(relPath string) (scope string, teamID *string) { - if !strings.HasPrefix(relPath, "teams/") { - return "personal", nil +// inferOwnerFromPath parses a tenant-relative path to determine ownership. +// Returns: agentID (*string), teamID (*string), scope (string), strippedPath (string). +// +// Path patterns: +// +// agents/{agent_key}/rest/of/path → agentID=lookup(key), scope="personal", path="rest/of/path" +// teams/{team_uuid}/rest/of/path → teamID=uuid, scope="team", path="rest/of/path" +// anything/else → scope="shared", path unchanged +// +// Returns scope="" to signal the file should be skipped (unknown agent or invalid team). +func inferOwnerFromPath(relPath string, agentMap map[string]string, teamSet map[string]bool) (agentID *string, teamID *string, scope string, strippedPath string) { + switch { + case strings.HasPrefix(relPath, "agents/"): + rest := relPath[len("agents/"):] + key, remainder, hasSlash := strings.Cut(rest, "/") + if !hasSlash || key == "" || strings.Contains(remainder, "..") { + return nil, nil, "", relPath // malformed or path traversal + } + agentUUID, ok := agentMap[key] + if !ok { + return nil, nil, "", relPath // unknown agent key → skip + } + return &agentUUID, nil, "personal", remainder + + case strings.HasPrefix(relPath, "teams/"): + rest := relPath[len("teams/"):] + id, remainder, hasSlash := strings.Cut(rest, "/") + if !hasSlash || id == "" || strings.Contains(remainder, "..") { + return nil, nil, "", relPath // malformed or path traversal + } + if _, parseErr := uuid.Parse(id); parseErr != nil { + return nil, nil, "", relPath // not a valid UUID → skip + } + if !teamSet[id] { + return nil, nil, "", relPath // team not found → skip + } + return nil, &id, "team", remainder + + default: + return nil, nil, "shared", relPath } - rest := relPath[len("teams/"):] - id, _, hasSlash := strings.Cut(rest, "/") - if !hasSlash || id == "" { - return "personal", nil - } - return "team", &id } // InferDocType guesses doc_type from path conventions. diff --git a/internal/vault/rescan_test.go b/internal/vault/rescan_test.go index 0c2ebbe4..2eb2428b 100644 --- a/internal/vault/rescan_test.go +++ b/internal/vault/rescan_test.go @@ -2,34 +2,112 @@ package vault import "testing" -func TestInferScopeFromPath(t *testing.T) { - tests := []struct { - path string - scope string - hasTeam bool - teamID string - }{ - {"notes/doc.md", "personal", false, ""}, - {"web-fetch/page.txt", "personal", false, ""}, - {"report.md", "personal", false, ""}, - {"teams/abc-def-123/report.md", "team", true, "abc-def-123"}, - {"teams/abc-def-123/deep/nested.md", "team", true, "abc-def-123"}, - {"teams/", "personal", false, ""}, // malformed, no team ID - {"teams", "personal", false, ""}, // no slash - {"teamsfoo/bar.md", "personal", false, ""}, // not teams/ prefix - {"telegram/123/teams/x/y.md", "personal", false, ""}, // teams not at root +// TestInferOwnerFromPath covers the new tenant-wide path parser. +func TestInferOwnerFromPath(t *testing.T) { + agentMap := map[string]string{ + "my-bot": "uuid-1", + "other-bot": "uuid-2", } + validUUID := "550e8400-e29b-41d4-a716-446655440000" + teamSet := map[string]bool{ + validUUID: true, + } + + tests := []struct { + path string + wantAgentID *string + wantTeamID *string + wantScope string + wantStrippedPath string + }{ + // agents/{key}/... → personal scope, strip prefix + { + path: "agents/my-bot/notes/todo.md", + wantAgentID: strPtr("uuid-1"), + wantScope: "personal", + wantStrippedPath: "notes/todo.md", + }, + // agents/{key} with no trailing path → personal, empty stripped path + { + path: "agents/my-bot/file.md", + wantAgentID: strPtr("uuid-1"), + wantScope: "personal", + wantStrippedPath: "file.md", + }, + // teams/{uuid}/... → team scope, strip prefix + { + path: "teams/" + validUUID + "/doc.md", + wantTeamID: strPtr(validUUID), + wantScope: "team", + wantStrippedPath: "doc.md", + }, + // teams/{uuid}/deep/nested → team scope + { + path: "teams/" + validUUID + "/deep/nested.md", + wantTeamID: strPtr(validUUID), + wantScope: "team", + wantStrippedPath: "deep/nested.md", + }, + // root-level file → shared scope, path unchanged + { + path: "README.md", + wantScope: "shared", + wantStrippedPath: "README.md", + }, + // nested file not under agents/ or teams/ → shared + { + path: "docs/guide.md", + wantScope: "shared", + wantStrippedPath: "docs/guide.md", + }, + // unknown agent → skip (scope="") + { + path: "agents/unknown-bot/file.md", + wantScope: "", + }, + // invalid team UUID → skip + { + path: "teams/not-a-uuid/file.md", + wantScope: "", + }, + // valid UUID but not in teamSet → skip + { + path: "teams/11111111-2222-3333-4444-555555555555/file.md", + wantScope: "", + }, + // malformed agents path (no trailing file) is still an unknown agent key check + { + path: "agents/unknown/", + wantScope: "", + }, + } + for _, tt := range tests { t.Run(tt.path, func(t *testing.T) { - scope, teamID := inferScopeFromPath(tt.path) - if scope != tt.scope { - t.Errorf("scope = %q, want %q", scope, tt.scope) + gotAgentID, gotTeamID, gotScope, gotPath := inferOwnerFromPath(tt.path, agentMap, teamSet) + + if gotScope != tt.wantScope { + t.Errorf("scope = %q, want %q", gotScope, tt.wantScope) } - if tt.hasTeam && (teamID == nil || *teamID != tt.teamID) { - t.Errorf("teamID = %v, want %q", teamID, tt.teamID) + if tt.wantScope == "" { + return // skip is signaled; remaining fields don't matter } - if !tt.hasTeam && teamID != nil { - t.Errorf("teamID = %v, want nil", teamID) + if tt.wantStrippedPath != "" && gotPath != tt.wantStrippedPath { + t.Errorf("strippedPath = %q, want %q", gotPath, tt.wantStrippedPath) + } + if tt.wantAgentID != nil { + if gotAgentID == nil || *gotAgentID != *tt.wantAgentID { + t.Errorf("agentID = %v, want %q", gotAgentID, *tt.wantAgentID) + } + } else if gotAgentID != nil { + t.Errorf("agentID = %v, want nil", gotAgentID) + } + if tt.wantTeamID != nil { + if gotTeamID == nil || *gotTeamID != *tt.wantTeamID { + t.Errorf("teamID = %v, want %q", gotTeamID, *tt.wantTeamID) + } + } else if gotTeamID != nil { + t.Errorf("teamID = %v, want nil", gotTeamID) } }) } @@ -78,3 +156,5 @@ func TestInferTitle(t *testing.T) { }) } } + +func strPtr(s string) *string { return &s } diff --git a/internal/vault/safe_walk.go b/internal/vault/safe_walk.go index 0fe813ba..a14e3c0a 100644 --- a/internal/vault/safe_walk.go +++ b/internal/vault/safe_walk.go @@ -158,16 +158,23 @@ func SafeWalkWorkspace(ctx context.Context, root string, opts WalkOptions) ([]Wa } // isExcludedDir returns true if an entire directory subtree should be skipped. +// relPath is the walk-relative path of the directory (e.g. "agents/my-bot/web-fetch"). func isExcludedDir(relPath string) bool { - first, _, _ := strings.Cut(relPath, "/") + // Get the directory's own name (last segment). + dirName := filepath.Base(relPath) - // Skip memory/ subtree entirely. - if first == "memory" { + // Skip memory/ at any depth. + if dirName == "memory" { return true } // Skip hidden dirs (. prefix) EXCEPT .uploads (user content). - if strings.HasPrefix(first, ".") && first != ".uploads" { + if strings.HasPrefix(dirName, ".") && dirName != ".uploads" { + return true + } + + // Skip web-fetch/ at any depth — content from external URLs may be dangerous. + if dirName == "web-fetch" { return true } @@ -189,19 +196,19 @@ var contextFiles = map[string]bool{ } // isExcludedPath returns true if a file should be excluded from vault registration. -// Defense-in-depth: also checks parent directory exclusions for callers that bypass dir walk. +// Defense-in-depth: checks ALL parent directory segments for exclusions. func isExcludedPath(relPath string) bool { - // Check first path segment for excluded directories. - first, _, _ := strings.Cut(relPath, "/") - - // memory/ prefix. - if first == "memory" { - return true - } - - // Hidden dirs (. prefix) except .uploads. - if strings.HasPrefix(first, ".") && first != ".uploads" { - return true + // Check every directory segment in the path. + dir := filepath.Dir(relPath) + for dir != "." && dir != "/" { + seg := filepath.Base(dir) + if seg == "memory" || seg == "web-fetch" { + return true + } + if strings.HasPrefix(seg, ".") && seg != ".uploads" { + return true + } + dir = filepath.Dir(dir) } base := filepath.Base(relPath) diff --git a/internal/vault/safe_walk_test.go b/internal/vault/safe_walk_test.go index e36895d4..f9f69b94 100644 --- a/internal/vault/safe_walk_test.go +++ b/internal/vault/safe_walk_test.go @@ -73,9 +73,9 @@ func TestSafeWalkWorkspace_ExcludedPaths(t *testing.T) { writeFile(t, dir, "data.db", "x") writeFile(t, dir, "data.db-wal", "x") writeFile(t, dir, "data.db-shm", "x") + writeFile(t, dir, "web-fetch/page.txt", "x") // excluded: external content // Should NOT be excluded writeFile(t, dir, "notes/meeting.md", "x") - writeFile(t, dir, "web-fetch/page.txt", "x") writeFile(t, dir, ".uploads/photo.jpg", "x") writeFile(t, dir, "soul-notes.md", "x") writeFile(t, dir, "deep/SOUL.md", "x") // not root level @@ -85,7 +85,7 @@ func TestSafeWalkWorkspace_ExcludedPaths(t *testing.T) { if err != nil { t.Fatal(err) } - if len(entries) != 6 { + if len(entries) != 5 { names := make([]string, len(entries)) for i, e := range entries { names[i] = e.RelPath @@ -201,7 +201,8 @@ func TestIsExcludedPath(t *testing.T) { {"data.db-shm", true}, // NOT excluded: {"notes/meeting.md", false}, - {"web-fetch/page.txt", false}, + {"web-fetch/page.txt", true}, + {"agents/my-bot/web-fetch/data.txt", true}, {"images/screenshot.png", false}, {"teams/abc-123/doc.md", false}, {"soul-notes.md", false}, diff --git a/migrations/000046_vault_nullable_agent_id.down.sql b/migrations/000046_vault_nullable_agent_id.down.sql new file mode 100644 index 00000000..4ed96cc0 --- /dev/null +++ b/migrations/000046_vault_nullable_agent_id.down.sql @@ -0,0 +1,18 @@ +-- Revert: delete orphaned docs, restore NOT NULL + CASCADE. +DELETE FROM vault_documents WHERE agent_id IS NULL; + +ALTER TABLE vault_documents ALTER COLUMN agent_id SET NOT NULL; + +ALTER TABLE vault_documents DROP CONSTRAINT vault_documents_agent_id_fkey; +ALTER TABLE vault_documents ADD CONSTRAINT vault_documents_agent_id_fkey + FOREIGN KEY (agent_id) REFERENCES agents(id) ON DELETE CASCADE; + +DROP INDEX IF EXISTS uq_vault_docs_agent_team_scope_path; +CREATE UNIQUE INDEX uq_vault_docs_agent_team_scope_path + ON vault_documents (agent_id, COALESCE(team_id, '00000000-0000-0000-0000-000000000000'), scope, path); + +DROP TRIGGER IF EXISTS trg_vault_docs_agent_null_scope ON vault_documents; +DROP FUNCTION IF EXISTS vault_docs_agent_null_scope_fix(); + +DROP INDEX IF EXISTS idx_vault_docs_agent_scope; +CREATE INDEX idx_vault_docs_agent_scope ON vault_documents(agent_id, scope); diff --git a/migrations/000046_vault_nullable_agent_id.up.sql b/migrations/000046_vault_nullable_agent_id.up.sql new file mode 100644 index 00000000..1ceb2b67 --- /dev/null +++ b/migrations/000046_vault_nullable_agent_id.up.sql @@ -0,0 +1,41 @@ +-- Make agent_id nullable so team-scoped and tenant-shared files can exist +-- without an owning agent. + +-- 1. Drop NOT NULL on agent_id. +ALTER TABLE vault_documents ALTER COLUMN agent_id DROP NOT NULL; + +-- 2. Change FK from CASCADE to SET NULL (agent deletion preserves docs). +ALTER TABLE vault_documents DROP CONSTRAINT vault_documents_agent_id_fkey; +ALTER TABLE vault_documents ADD CONSTRAINT vault_documents_agent_id_fkey + FOREIGN KEY (agent_id) REFERENCES agents(id) ON DELETE SET NULL; + +-- 3. Replace unique index: add tenant_id as leading column, COALESCE both nullable cols. +DROP INDEX IF EXISTS uq_vault_docs_agent_team_scope_path; +CREATE UNIQUE INDEX uq_vault_docs_agent_team_scope_path + ON vault_documents ( + tenant_id, + COALESCE(agent_id, '00000000-0000-0000-0000-000000000000'), + COALESCE(team_id, '00000000-0000-0000-0000-000000000000'), + scope, + path + ); + +-- 4. Trigger: when agent deleted (SET NULL) and no team -> scope='shared'. +CREATE OR REPLACE FUNCTION vault_docs_agent_null_scope_fix() +RETURNS TRIGGER AS $$ +BEGIN + IF NEW.agent_id IS NULL AND OLD.agent_id IS NOT NULL AND NEW.team_id IS NULL THEN + NEW.scope := 'shared'; + END IF; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER trg_vault_docs_agent_null_scope + BEFORE UPDATE OF agent_id ON vault_documents + FOR EACH ROW + EXECUTE FUNCTION vault_docs_agent_null_scope_fix(); + +-- 5. Partial index for agent-scoped queries (skip NULLs). +DROP INDEX IF EXISTS idx_vault_docs_agent_scope; +CREATE INDEX idx_vault_docs_agent_scope ON vault_documents(agent_id, scope) WHERE agent_id IS NOT NULL; diff --git a/ui/web/src/components/graph/sigma-graph-container.tsx b/ui/web/src/components/graph/sigma-graph-container.tsx index 3aa73128..613eeb6d 100644 --- a/ui/web/src/components/graph/sigma-graph-container.tsx +++ b/ui/web/src/components/graph/sigma-graph-container.tsx @@ -56,6 +56,8 @@ export function SigmaGraphContainer({ }: SigmaGraphContainerProps) { const containerRef = useRef(null); const internalSigmaRef = useRef(null); + // Incremented when sigma instance changes — used to trigger event handler registration. + const [sigmaVersion, setSigmaVersion] = useState(0); const [hoveredNode, setHoveredNode] = useState(null); // Pulse phase for animated highlighted edges (0..1, cycles) const [pulsePhase, setPulsePhase] = useState(0); @@ -64,6 +66,7 @@ export function SigmaGraphContainer({ const setSigmaRef = useCallback( (instance: Sigma | null) => { internalSigmaRef.current = instance; + setSigmaVersion((v) => v + 1); onSigmaReady?.(instance); }, [onSigmaReady], @@ -337,7 +340,15 @@ export function SigmaGraphContainer({ sigma.refresh({ skipIndexation: true }); }, [pulsePhase]); - // --- Event handlers: use Sigma v3 native doubleClickNode (snappier than 280ms timer) --- + // --- Event handlers: use refs for values that change frequently to avoid + // re-registering Sigma listeners (which drops in-flight double-click events) --- + const selectedNodeIdRef = useRef(selectedNodeId); + selectedNodeIdRef.current = selectedNodeId; + const onNodeSelectRef = useRef(onNodeSelect); + onNodeSelectRef.current = onNodeSelect; + const onNodeDoubleClickRef = useRef(onNodeDoubleClick); + onNodeDoubleClickRef.current = onNodeDoubleClick; + useEffect(() => { const sigma = internalSigmaRef.current; if (!sigma) return; @@ -353,17 +364,17 @@ export function SigmaGraphContainer({ }; const handleClickNode = ({ node }: { node: string }) => { - onNodeSelect?.(node === selectedNodeId ? null : node); + onNodeSelectRef.current?.(node === selectedNodeIdRef.current ? null : node); }; const handleDoubleClickNode = ({ node, event }: { node: string; event: { preventSigmaDefault?: () => void } }) => { // Prevent Sigma's default zoom-in behavior on double-click event.preventSigmaDefault?.(); - onNodeDoubleClick?.(node); + onNodeDoubleClickRef.current?.(node); }; const handleClickStage = () => { - onNodeSelect?.(null); + onNodeSelectRef.current?.(null); }; sigma.on("enterNode", handleEnterNode); @@ -379,7 +390,7 @@ export function SigmaGraphContainer({ sigma.off("doubleClickNode", handleDoubleClickNode); sigma.off("clickStage", handleClickStage); }; - }, [selectedNodeId, onNodeSelect, onNodeDoubleClick]); + }, [sigmaVersion]); // re-register only when sigma instance changes // NOTE: Click on node NO LONGER moves camera. // Camera only animates for explicit user actions (search, fit-to-view, keyboard F). diff --git a/ui/web/src/components/layout/system-settings-constants.ts b/ui/web/src/components/layout/system-settings-constants.ts index 504310ea..3bb1226c 100644 --- a/ui/web/src/components/layout/system-settings-constants.ts +++ b/ui/web/src/components/layout/system-settings-constants.ts @@ -42,6 +42,8 @@ export interface InitState { kgProvider: string; kgModel: string; kgMinConfidence: string; + bgProvider: string; + bgModel: string; } export const DEFAULTS: InitState = { @@ -51,6 +53,7 @@ export const DEFAULTS: InitState = { compProvider: "", compModel: "", compThreshold: "", compKeepRecent: "", compMaxTokens: "", kgProvider: "", kgModel: "", kgMinConfidence: "0.75", + bgProvider: "", bgModel: "", }; export function parseBool(v: string | undefined, fallback: boolean): boolean { diff --git a/ui/web/src/components/layout/system-settings-modal.tsx b/ui/web/src/components/layout/system-settings-modal.tsx index ce36a27c..01cc4815 100644 --- a/ui/web/src/components/layout/system-settings-modal.tsx +++ b/ui/web/src/components/layout/system-settings-modal.tsx @@ -1,6 +1,6 @@ import { useState, useEffect, useCallback } from "react"; import { useTranslation } from "react-i18next"; -import { Settings2, Loader2, Save, AlertTriangle, Info, ExternalLink, Network } from "lucide-react"; +import { Settings2, Loader2, Save, AlertTriangle, Info, ExternalLink, Network, Cog } from "lucide-react"; import { Link } from "react-router"; import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/ui/dialog"; import { Button } from "@/components/ui/button"; @@ -57,6 +57,10 @@ export function SystemSettingsModal({ open, onOpenChange }: SystemSettingsModalP const [kgModel, setKgModel] = useState(""); const [kgMinConfidence, setKgMinConfidence] = useState("0.75"); + // Background Workers + const [bgProvider, setBgProvider] = useState(""); + const [bgModel, setBgModel] = useState(""); + const applyConfigs = useCallback(( configs: Record, kgSettings?: { extraction_provider?: string; extraction_model?: string; min_confidence?: number }, @@ -71,12 +75,14 @@ export function SystemSettingsModal({ open, onOpenChange }: SystemSettingsModalP compMaxTokens: configs["compaction.max_tokens"] ?? "", kgProvider: kgSettings?.extraction_provider ?? "", kgModel: kgSettings?.extraction_model ?? "", kgMinConfidence: String(kgSettings?.min_confidence ?? 0.75), + bgProvider: configs["background.provider"] ?? "", bgModel: configs["background.model"] ?? "", }; setInit(s); setEmbProvider(s.embProvider); setEmbModel(s.embModel); setEmbMaxChunkLen(s.embMaxChunkLen); setEmbChunkOverlap(s.embChunkOverlap); setToolStatus(s.toolStatus); setBlockReply(s.blockReply); setIntentClassify(s.intentClassify); setCompProvider(s.compProvider); setCompModel(s.compModel); setCompThreshold(s.compThreshold); setCompKeepRecent(s.compKeepRecent); setCompMaxTokens(s.compMaxTokens); setKgProvider(s.kgProvider); setKgModel(s.kgModel); setKgMinConfidence(s.kgMinConfidence); + setBgProvider(s.bgProvider); setBgModel(s.bgModel); resetEmb(); }, [resetEmb]); @@ -118,6 +124,8 @@ export function SystemSettingsModal({ open, onOpenChange }: SystemSettingsModalP if (compThreshold !== init.compThreshold) updates["compaction.threshold"] = compThreshold; if (compKeepRecent !== init.compKeepRecent) updates["compaction.keep_recent"] = compKeepRecent; if (compMaxTokens !== init.compMaxTokens) updates["compaction.max_tokens"] = compMaxTokens; + if (bgProvider !== init.bgProvider) updates["background.provider"] = bgProvider; + if (bgModel !== init.bgModel) updates["background.model"] = bgModel; for (const [key, value] of Object.entries(updates)) await http.put(`/v1/system-configs/${key}`, { value }); const kgChanged = kgProvider !== init.kgProvider || kgModel !== init.kgModel || kgMinConfidence !== init.kgMinConfidence; if (kgChanged) { @@ -179,6 +187,20 @@ export function SystemSettingsModal({ open, onOpenChange }: SystemSettingsModalP + {/* Background Workers */} + + + {t("bg.title")} + {t("bg.description")} + + + { setBgProvider(v); setBgModel(""); }} model={bgModel} onModelChange={setBgModel} allowEmpty providerLabel={t("bg.provider")} modelLabel={t("bg.model")} providerTip={t("bg.providerTip")} modelTip={t("bg.modelTip")} providerPlaceholder={t("bg.providerPlaceholder")} modelPlaceholder={t("bg.modelPlaceholder")} /> +
+ {t("bg.info")} +
+ + + c.value === configType)?.descKey ?? ""; return ( -
+
{/* Header */}
@@ -305,6 +305,6 @@ export function AgentPermissionsTab({ agentId }: AgentPermissionsTabProps) { )}
)} -
+
); } diff --git a/ui/web/src/pages/agents/agent-detail/evolution-tab/evolution-metrics-charts.tsx b/ui/web/src/pages/agents/agent-detail/evolution-tab/evolution-metrics-charts.tsx index 821ee035..abedc644 100644 --- a/ui/web/src/pages/agents/agent-detail/evolution-tab/evolution-metrics-charts.tsx +++ b/ui/web/src/pages/agents/agent-detail/evolution-tab/evolution-metrics-charts.tsx @@ -18,16 +18,20 @@ export function EvolutionMetricsCharts({ toolAggs, retrievalAggs, loading }: Evo return
; } + // API returns rates as 0-1 fractions; convert to 0-100 for chart display. + const toolData = toolAggs.map((a) => ({ ...a, success_rate: a.success_rate * 100 })); + const retrievalData = retrievalAggs.map((a) => ({ ...a, usage_rate: a.usage_rate * 100 })); + return (
{/* Tool Success Rate */}

{t("detail.evolution.toolSuccess")}

- {toolAggs.length === 0 ? ( + {toolData.length === 0 ? (

{t("detail.evolution.noMetrics")}

) : ( - + `${v}%`} /> @@ -35,12 +39,12 @@ export function EvolutionMetricsCharts({ toolAggs, retrievalAggs, loading }: Evo formatter={(value, _name, props) => { const v = Number(value ?? 0); const p = props?.payload as ToolAggregate | undefined; - return [`${v.toFixed(1)}%`, p ? `${p.call_count} calls, ${p.avg_duration_ms.toFixed(0)}ms avg` : ""]; + return [`${v.toFixed(1)}%`, p ? t("detail.evolution.tooltipCalls", { count: p.call_count, ms: p.avg_duration_ms.toFixed(0) }) : ""]; }} />

{t("detail.evolution.retrievalQuality")}

- {retrievalAggs.length === 0 ? ( + {retrievalData.length === 0 ? (

{t("detail.evolution.noMetrics")}

) : ( - + `${v}%`} /> @@ -65,12 +69,12 @@ export function EvolutionMetricsCharts({ toolAggs, retrievalAggs, loading }: Evo formatter={(value, _name, props) => { const v = Number(value ?? 0); const p = props?.payload as RetrievalAggregate | undefined; - return [`${v.toFixed(1)}%`, p ? `${p.query_count} queries, score ${p.avg_score.toFixed(2)}` : ""]; + return [`${v.toFixed(1)}%`, p ? t("detail.evolution.tooltipQueries", { count: p.query_count, score: p.avg_score.toFixed(2) }) : ""]; }} /> - Type - Suggestion - Status - Created - Actions + {t("detail.evolution.colType")} + {t("detail.evolution.colSuggestion")} + {t("detail.evolution.colStatus")} + {t("detail.evolution.colCreated")} + {t("detail.evolution.colActions")} @@ -144,10 +144,10 @@ export function EvolutionSuggestionsTable({ suggestions, loading, onUpdateStatus diff --git a/ui/web/src/pages/vault/hooks/use-vault-mutations.ts b/ui/web/src/pages/vault/hooks/use-vault-mutations.ts index c5561ab6..208752c5 100644 --- a/ui/web/src/pages/vault/hooks/use-vault-mutations.ts +++ b/ui/web/src/pages/vault/hooks/use-vault-mutations.ts @@ -107,17 +107,16 @@ export function useCreateLink(agentId: string) { return { create }; } -/** Rescan workspace to sync vault documents from filesystem. */ -export function useRescanWorkspace(agentId: string) { +/** Rescan workspace to sync vault documents from filesystem (tenant-wide). */ +export function useRescanWorkspace() { const http = useHttp(); const queryClient = useQueryClient(); const [isPending, setIsPending] = useState(false); const rescan = useCallback(async () => { - if (!agentId) return; setIsPending(true); try { - const result = await http.post(`/v1/agents/${agentId}/vault/rescan`, {}); + const result = await http.post(`/v1/vault/rescan`, {}); await queryClient.invalidateQueries({ queryKey: [VAULT_KEY] }); const parts: string[] = []; @@ -140,7 +139,7 @@ export function useRescanWorkspace(agentId: string) { } finally { setIsPending(false); } - }, [http, agentId, queryClient]); + }, [http, queryClient]); return { rescan, isPending }; } diff --git a/ui/web/src/pages/vault/hooks/use-vault.ts b/ui/web/src/pages/vault/hooks/use-vault.ts index 9e202c36..4aeff68f 100644 --- a/ui/web/src/pages/vault/hooks/use-vault.ts +++ b/ui/web/src/pages/vault/hooks/use-vault.ts @@ -1,4 +1,4 @@ -import { useCallback, useMemo } from "react"; +import { useCallback, useEffect, useMemo, useState } from "react"; import { useQuery, useQueryClient } from "@tanstack/react-query"; import { useHttp } from "@/hooks/use-ws"; import type { VaultDocument, VaultLink, VaultSearchResult } from "@/types/vault"; @@ -72,6 +72,26 @@ export function useVaultSearch(agentId: string) { return { search, invalidate }; } +/** Tenant-wide vault search (agent_id optional). */ +export function useVaultSearchAll() { + const http = useHttp(); + + const search = useCallback( + async (query: string, opts?: { agentId?: string; docTypes?: string[]; teamId?: string; maxResults?: number }) => { + return http.post("/v1/vault/search", { + query, + agent_id: opts?.agentId || undefined, + doc_types: opts?.docTypes, + team_id: opts?.teamId || undefined, + max_results: opts?.maxResults ?? 20, + }); + }, + [http], + ); + + return { search }; +} + /** Fetch all links for a set of vault documents (for graph view). */ export function useVaultAllLinks(agentId: string, documents: { id: string }[]) { const http = useHttp(); @@ -219,6 +239,25 @@ export function useVaultFileContent(path: string | null) { return { content: data?.content ?? null, size: data?.size ?? 0, loading: isLoading, error: !!error }; } +/** Fetch an image file as blob URL for authenticated rendering in tags. */ +export function useVaultImageUrl(path: string | null): { url: string | null; error: boolean } { + const http = useHttp(); + const [url, setUrl] = useState(null); + const [error, setError] = useState(false); + + useEffect(() => { + if (!path) { setUrl(null); setError(false); return; } + let revoke: string | null = null; + setError(false); + http.downloadBlob(`/v1/storage/files/${encodeURIComponent(path)}?raw=true`) + .then((blob) => { revoke = URL.createObjectURL(blob); setUrl(revoke); }) + .catch(() => { setUrl(null); setError(true); }); + return () => { if (revoke) URL.revokeObjectURL(revoke); }; + }, [path, http]); + + return { url, error }; +} + // Re-export mutations for convenience — consumers can import from this single file export { useCreateDocument, diff --git a/ui/web/src/pages/vault/vault-detail-dialog.tsx b/ui/web/src/pages/vault/vault-detail-dialog.tsx index 102a0092..f2b1367f 100644 --- a/ui/web/src/pages/vault/vault-detail-dialog.tsx +++ b/ui/web/src/pages/vault/vault-detail-dialog.tsx @@ -1,6 +1,6 @@ -import { useState } from "react"; +import { useState, useMemo } from "react"; import { useTranslation } from "react-i18next"; -import { Pencil, Plus, FileText, Link2 } from "lucide-react"; +import { Pencil, Plus, FileText, Link2, FileQuestion } from "lucide-react"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; @@ -8,7 +8,7 @@ import { Dialog, DialogContent, DialogHeader, DialogTitle, } from "@/components/ui/dialog"; import { MarkdownRenderer } from "@/components/shared/markdown-renderer"; -import { useVaultLinks, useVaultFileContent, useUpdateDocument, useDeleteDocument } from "./hooks/use-vault"; +import { useVaultLinks, useVaultFileContent, useVaultImageUrl, useUpdateDocument, useDeleteDocument } from "./hooks/use-vault"; import { VaultLinkDialog } from "./vault-link-dialog"; import { VaultEditControls, DocTypeSelect, ScopeSelect, LinkBadge, @@ -34,10 +34,22 @@ export function VaultDetailDialog({ doc, open, onOpenChange, onDeleted }: Props) const [confirmDelete, setConfirmDelete] = useState(false); const [linkDialogOpen, setLinkDialogOpen] = useState(false); - // Content always loaded when dialog is open + // Determine if this is an image that can be rendered. + const isImage = useMemo(() => { + const mime = doc?.metadata?.mime_type as string | undefined; + if (mime?.startsWith("image/")) return true; + const ext = doc?.path.split(".").pop()?.toLowerCase() ?? ""; + return ["png", "jpg", "jpeg", "gif", "webp", "svg", "bmp", "ico"].includes(ext); + }, [doc?.metadata, doc?.path]); + + const isMedia = doc?.doc_type === "media"; + + // Only fetch text content for non-media files (media/binary cannot be rendered as markdown). const { content: fileContent, loading: contentLoading, error: contentError } = useVaultFileContent( - open && doc ? doc.path : null, + open && doc && !isMedia ? doc.path : null, ); + // Fetch image as authenticated blob URL for rendering. + const { url: imageUrl, error: imageError } = useVaultImageUrl(open && isMedia && isImage && doc ? doc.path : null); const { update } = useUpdateDocument(doc?.agent_id ?? "", doc?.id ?? ""); const { remove: removeDoc } = useDeleteDocument(doc?.agent_id ?? "", doc?.id ?? ""); @@ -149,7 +161,37 @@ export function VaultDetailDialog({ doc, open, onOpenChange, onDeleted }: Props) {/* Content preview — always visible, scrollable */}
- {contentLoading ? ( + {isMedia ? ( + isImage ? ( +
+ {imageUrl ? ( + {doc.title + ) : imageError ? ( +
+ + {t("detail.fileNotFound")} +
+ ) : ( +
+ )} +
+ ) : ( +
+ +
+

{t("detail.binaryFile")}

+

{(doc.metadata?.mime_type as string) || doc.path.split(".").pop()?.toUpperCase()}

+
+ {doc.summary && ( +

{doc.summary}

+ )} +
+ ) + ) : contentLoading ? (
diff --git a/ui/web/src/pages/vault/vault-document-sidebar.tsx b/ui/web/src/pages/vault/vault-document-sidebar.tsx index 380864b1..ae1a154e 100644 --- a/ui/web/src/pages/vault/vault-document-sidebar.tsx +++ b/ui/web/src/pages/vault/vault-document-sidebar.tsx @@ -1,24 +1,26 @@ -import { useEffect, useRef } from "react"; +import { useState, useEffect, useRef, useCallback } from "react"; import { useTranslation } from "react-i18next"; import { - Link2, ChevronLeft, ChevronRight, + Link2, ChevronLeft, ChevronRight, Search, X, Loader2, FileText, Brain, StickyNote, Sparkles, Clock, Image, } from "lucide-react"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { formatRelativeTime } from "@/lib/format"; -import type { VaultDocument } from "@/types/vault"; +import type { VaultDocument, VaultSearchResult } from "@/types/vault"; +import { useVaultSearchAll } from "./hooks/use-vault"; -const DOC_TYPE_CONFIG: Record = { - context: { color: "text-blue-600 dark:text-blue-400", bg: "bg-blue-500/10", icon: FileText }, - memory: { color: "text-purple-600 dark:text-purple-400", bg: "bg-purple-500/10", icon: Brain }, - note: { color: "text-amber-600 dark:text-amber-400", bg: "bg-amber-500/10", icon: StickyNote }, - skill: { color: "text-emerald-600 dark:text-emerald-400", bg: "bg-emerald-500/10", icon: Sparkles }, - episodic: { color: "text-orange-600 dark:text-orange-400", bg: "bg-orange-500/10", icon: Clock }, - media: { color: "text-rose-600 dark:text-rose-400", bg: "bg-rose-500/10", icon: Image }, +const DOC_TYPE_CONFIG: Record = { + context: { color: "text-blue-600 dark:text-blue-400", bg: "bg-blue-500/10", icon: FileText, dotColor: "bg-blue-500" }, + memory: { color: "text-purple-600 dark:text-purple-400", bg: "bg-purple-500/10", icon: Brain, dotColor: "bg-purple-500" }, + note: { color: "text-amber-600 dark:text-amber-400", bg: "bg-amber-500/10", icon: StickyNote, dotColor: "bg-amber-500" }, + skill: { color: "text-emerald-600 dark:text-emerald-400", bg: "bg-emerald-500/10", icon: Sparkles, dotColor: "bg-emerald-500" }, + episodic: { color: "text-orange-600 dark:text-orange-400", bg: "bg-orange-500/10", icon: Clock, dotColor: "bg-orange-500" }, + media: { color: "text-rose-600 dark:text-rose-400", bg: "bg-rose-500/10", icon: Image, dotColor: "bg-rose-500" }, }; -const DEFAULT_CONFIG = { color: "text-muted-foreground", bg: "bg-muted", icon: FileText }; +const DEFAULT_CONFIG = { color: "text-muted-foreground", bg: "bg-muted", icon: FileText, dotColor: "bg-muted-foreground" }; +const DOC_TYPES = ["context", "memory", "note", "skill", "episodic", "media"] as const; interface Props { documents: VaultDocument[]; @@ -30,6 +32,10 @@ interface Props { totalPages: number; total: number; onPageChange: (page: number) => void; + docType: string; + onDocTypeChange: (type: string) => void; + agentId: string; + teamId: string; } function DocCard({ doc, selected, linkCount, onClick }: { @@ -54,17 +60,13 @@ function DocCard({ doc, selected, linkCount, onClick }: { }`} onClick={onClick} > - {/* Type icon */}
- - {/* Content */}
{doc.title || doc.path.split("/").pop()} -
{t(`type.${doc.doc_type}`)} · @@ -87,21 +89,125 @@ function DocCard({ doc, selected, linkCount, onClick }: { } export function VaultDocumentSidebar({ - documents, selectedId, linkCounts, onSelect, loading, page, totalPages, total, onPageChange, + documents, selectedId, linkCounts, onSelect, loading, + page, totalPages, total, onPageChange, + docType, onDocTypeChange, agentId, teamId, }: Props) { const { t } = useTranslation("vault"); + const [query, setQuery] = useState(""); + const [searchResults, setSearchResults] = useState(null); + const [searching, setSearching] = useState(false); + const inputRef = useRef(null); + const debounceRef = useRef>(null); + const { search } = useVaultSearchAll(); + + const doSearch = useCallback( + async (q: string) => { + if (!q.trim()) { + setSearchResults(null); + return; + } + setSearching(true); + try { + const results = await search(q, { + agentId: agentId || undefined, + docTypes: docType ? [docType] : undefined, + teamId: teamId || undefined, + maxResults: 30, + }); + setSearchResults(results); + } catch { + setSearchResults([]); + } finally { + setSearching(false); + } + }, + [search, agentId, docType, teamId], + ); + + const handleQueryChange = (value: string) => { + setQuery(value); + if (debounceRef.current) clearTimeout(debounceRef.current); + if (!value.trim()) { + setSearchResults(null); + return; + } + debounceRef.current = setTimeout(() => doSearch(value), 300); + }; + + const clearSearch = () => { + setQuery(""); + setSearchResults(null); + inputRef.current?.focus(); + }; + + // Show search results or regular document list + const isSearchMode = query.trim().length > 0; + const displayDocs = isSearchMode && searchResults + ? searchResults.map((r) => r.document) + : documents; return (
- {/* Header */} -
- {t("title")} - {total} + {/* Filter bar: type chips + search */} +
+ {/* Type filter chips — horizontal scroll */} +
+ + {DOC_TYPES.map((dt) => { + const cfg = DOC_TYPE_CONFIG[dt] ?? DEFAULT_CONFIG; + const active = docType === dt; + return ( + + ); + })} + {total} +
+ + {/* Search input */} +
+ {searching + ? + : + } + handleQueryChange(e.target.value)} + placeholder={t("searchPlaceholder")} + className="w-full rounded-md border bg-background pl-7 pr-7 py-1 text-base md:text-xs placeholder:text-muted-foreground focus:outline-none focus:ring-1 focus:ring-ring" + /> + {query && ( + + )} +
{/* Doc list */}
- {loading && documents.length === 0 ? ( + {(loading || searching) && displayDocs.length === 0 ? ( Array.from({ length: 6 }).map((_, i) => (
@@ -111,13 +217,13 @@ export function VaultDocumentSidebar({
)) - ) : documents.length === 0 ? ( + ) : displayDocs.length === 0 ? (
- {t("noDocuments")} + {isSearchMode ? t("noResults") : t("noDocuments")}
) : ( - documents.map((doc) => ( + displayDocs.map((doc) => ( - {/* Pagination footer */} - {totalPages > 1 && ( + {/* Pagination footer — hidden during search */} + {!isSearchMode && totalPages > 1 && (
@@ -136,13 +143,11 @@ export function VaultPage() { - - - + - {!selectedAgent ? t("selectAgentFirst", "Select an agent first") : t("rescanTooltip", "Rescan workspace")} + {t("rescanTooltip", "Rescan workspace")}